From 453a5e569bc3f5ee184617df8a241cb5a4b6bfc8 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Tue, 22 Sep 2026 12:58:14 -0600 Subject: [PATCH 1/3] Split ESMF weight generation into dedicated steps for ISMIP7 forcing ESMF_RegridWeightGen benefits from many MPI tasks (128-512) but the rest of forcing processing runs efficiently on a single node. This change separates weight-generation into BuildMappingFile steps that can request large node allocations while processing steps run at ntasks=1, avoiding waste of hundreds of node-hours. Additionally, mapping files are now reusable across scenarios since they depend only on ice_sheet + mali_mesh_name + method_remap. Users can set mapping_files_path in the config to reuse weights from previous runs, and all built weights are automatically copied to output_base_path/mapping_files/ for future reuse. Changes: - Add BuildMappingFile step to atmosphere, ocean_thermal, and fracture test cases as first step (runs at esmf_ntasks) - Update all processing steps to consume weights via symlinks from ../build_mapping_file/map_*.nc instead of building them inline - Add optional mapping_files_path config option to [ismip7] section for cross-run weight reuse - Remove build_mapping_file calls from all processing step run() methods and add weight files as input dependencies in setup() This enables allocating large node counts only for the 5-10 minute weight-generation step while running the hours-long processing steps on single nodes. Co-Authored-By: Claude Sonnet 4.5 --- .../ismip7_forcing/atmosphere/__init__.py | 4 + .../atmosphere/build_mapping_file.py | 132 +++++++++++++++ .../atmosphere/process_runoff.py | 23 +-- .../ismip7_forcing/atmosphere/process_smb.py | 23 +-- .../atmosphere/process_smb_gradient.py | 23 +-- .../atmosphere/process_temperature.py | 23 +-- .../process_temperature_gradient.py | 23 +-- .../tests/ismip7_forcing/fracture/__init__.py | 4 + .../fracture/build_mapping_file.py | 142 ++++++++++++++++ .../fracture/process_excess_melt.py | 25 +-- .../fracture/process_lake_properties.py | 26 +-- .../fracture/process_shelf_collapse.py | 24 +-- .../tests/ismip7_forcing/ismip7_forcing.cfg | 7 + .../ismip7_forcing/ismip7_forcing_ocx_ais.cfg | 7 + .../ismip7_forcing/ismip7_forcing_ocx_gis.cfg | 7 + .../ismip7_forcing/ismip7_forcing_test.cfg | 7 + .../ismip7_forcing/ocean_thermal/__init__.py | 4 + .../ocean_thermal/build_mapping_file.py | 152 ++++++++++++++++++ .../ocean_thermal/process_thermal_forcing.py | 30 ++-- 19 files changed, 586 insertions(+), 100 deletions(-) create mode 100644 compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py create mode 100644 compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py create mode 100644 compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py index 7e577b2026..6c49cb98ad 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py @@ -1,3 +1,6 @@ +from compass.landice.tests.ismip7_forcing.atmosphere.build_mapping_file import ( # noqa: E501 + BuildMappingFile, +) from compass.landice.tests.ismip7_forcing.atmosphere.process_runoff import ( ProcessRunoff, ) @@ -40,6 +43,7 @@ def __init__(self, test_group): subdir = name super().__init__(test_group=test_group, name=name, subdir=subdir) + self.add_step(BuildMappingFile(test_case=self)) self.add_step(ProcessSmb(test_case=self)) self.add_step(ProcessTemperature(test_case=self)) self.add_step(ProcessSmbGradient(test_case=self)) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py new file mode 100644 index 0000000000..fd1df2c948 --- /dev/null +++ b/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py @@ -0,0 +1,132 @@ +import os +import shutil + +from compass.landice.ismip7.ice_sheet_params import get_params +from compass.landice.ismip7.mapping import build_mapping_file +from compass.step import Step + + +class BuildMappingFile(Step): + """ + A step for building the ESMF mapping file (regridding weights) from + the ISMIP7 atmosphere grid to the MALI mesh. This step runs + ESMF_RegridWeightGen with the full esmf_ntasks allocation, so it can + be run on a separate node allocation from the processing steps (which + run on a single node). + + The mapping file can be reused across scenarios, since it depends only on + the ice sheet, mesh name, and remapping method. If mapping_files_path is + provided in the config, this step will symlink any existing weight file + from that directory and skip the build. + """ + + def __init__(self, test_case): + """ + Create the step + + Parameters + ---------- + test_case : compass.landice.tests.ismip7_forcing.atmosphere.Atmosphere + The test case this step belongs to + """ + super().__init__(test_case=test_case, name="build_mapping_file") + + def setup(self): + """ + Set up this step of the test case + """ + config = self.config + section = config["ismip7"] + base_path_mali = section.get("base_path_mali") + mali_mesh_file = section.get("mali_mesh_file") + + self.add_input_file(filename=mali_mesh_file, + target=os.path.join(base_path_mali, + mali_mesh_file)) + + # Request the full esmf_ntasks allocation for this step + self.ntasks = section.getint("esmf_ntasks") + self.min_tasks = self.ntasks + + def run(self): + """ + Run this step of the test case + """ + logger = self.logger + config = self.config + params = get_params(config) + + section = config["ismip7"] + base_path_ismip7 = section.get("base_path_ismip7") + mali_mesh_name = section.get("mali_mesh_name") + mali_mesh_file = section.get("mali_mesh_file") + ice_sheet = section.get("ice_sheet") + output_base_path = section.get("output_base_path") + + # Check if user provided a path to reuse existing weight files + mapping_files_path = section.get("mapping_files_path") + + section = config["ismip7_atmosphere"] + method_remap = section.get("method_remap") + + # Construct the canonical mapping file name + mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" + f"{mali_mesh_name}_{method_remap}.nc") + + # If mapping_files_path is provided, symlink any existing file + if mapping_files_path != "NotAvailable": + source_file = os.path.join(mapping_files_path, mapping_file) + if os.path.exists(source_file): + logger.info(f"Symlinking existing mapping file from " + f"{mapping_files_path}") + if os.path.exists(mapping_file): + os.remove(mapping_file) + os.symlink(source_file, mapping_file) + + # Use any atmosphere file as the grid template for building the + # mapping file. We'll use the first year's acabf (SMB) file. + prefix = params['prefix'] + resolution = params['atm_resolution'] + version = params['atm_version'] + model = (section.get("model") if params['atm_model'] is None + else params['atm_model']) + scenario = section.get("scenario") + + input_path = os.path.join(base_path_ismip7, "acabf", version) + # Use a simple pattern that will match the first available file + file_pattern = (f"acabf_{prefix}_{model}_{scenario}_" + f"SDBN1-{resolution}_{version}_*.nc") + ismip7_grid_file = os.path.join(input_path, file_pattern) + + import glob + grid_files = sorted(glob.glob(ismip7_grid_file)) + if not grid_files: + raise FileNotFoundError( + f"No atmosphere grid file found matching pattern:\n" + f" {ismip7_grid_file}") + ismip7_grid_file = grid_files[0] + logger.info(f"Using grid template: " + f"{os.path.basename(ismip7_grid_file)}") + + # Build the mapping file (build_mapping_file will skip if it already + # exists, e.g., from the symlink above or a previous failed run) + build_mapping_file(config, logger, ismip7_grid_file, mapping_file, + mali_mesh_file=mali_mesh_file, + method_remap=method_remap) + + # Copy the mapping file to output_base_path/mapping_files/ for reuse + # in future runs + mapping_files_dir = os.path.join(output_base_path, "mapping_files") + if not os.path.exists(mapping_files_dir): + os.makedirs(mapping_files_dir) + + dst = os.path.join(mapping_files_dir, mapping_file) + # Only copy if it's a real file (not a symlink we just created) + if not os.path.islink(mapping_file): + logger.info(f"Copying mapping file to {mapping_files_dir} for reuse") + shutil.copy(mapping_file, dst) + else: + logger.info(f"Mapping file is a symlink; not copying to " + f"{mapping_files_dir}") + + logger.info("Done building atmosphere mapping file.") diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py index 8125abe250..39282d1955 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py @@ -7,7 +7,6 @@ from mpas_tools.logging import check_call from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -38,11 +37,22 @@ def setup(self): section = config["ismip7"] base_path_mali = section.get("base_path_mali") mali_mesh_file = section.get("mali_mesh_file") + mali_mesh_name = section.get("mali_mesh_name") + ice_sheet = section.get("ice_sheet") + + section_atm = config["ismip7_atmosphere"] + method_remap = section_atm.get("method_remap") self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) + # Add input file for the mapping file from build_mapping_file step + mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" + f"{mali_mesh_name}_{method_remap}.nc") + self.add_input_file(filename=mapping_file, + target=f"../build_mapping_file/{mapping_file}") + def run(self): """ Run this step of the test case @@ -54,7 +64,6 @@ def run(self): section = config["ismip7"] base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -103,17 +112,11 @@ def run(self): logger.info(f"Found {len(input_files)} runoff files for years " f"{start_year}-{end_year}") - # Build mapping file (reuse if already created by other atm steps) + # Construct the mapping file name (symlinked from + # build_mapping_file step) mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" f"{mali_mesh_name}_{method_remap}.nc") - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each year file remapped_files = [] for input_file in input_files: diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py index 9701265638..0b2aed2a54 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py @@ -7,7 +7,6 @@ from mpas_tools.logging import check_call from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -38,11 +37,22 @@ def setup(self): section = config["ismip7"] base_path_mali = section.get("base_path_mali") mali_mesh_file = section.get("mali_mesh_file") + mali_mesh_name = section.get("mali_mesh_name") + ice_sheet = section.get("ice_sheet") + + section_atm = config["ismip7_atmosphere"] + method_remap = section_atm.get("method_remap") self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) + # Add input file for the mapping file from build_mapping_file step + mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" + f"{mali_mesh_name}_{method_remap}.nc") + self.add_input_file(filename=mapping_file, + target=f"../build_mapping_file/{mapping_file}") + def run(self): """ Run this step of the test case @@ -54,7 +64,6 @@ def run(self): section = config["ismip7"] base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -102,18 +111,12 @@ def run(self): logger.info(f"Found {len(input_files)} SMB files for years " f"{start_year}-{end_year}") - # Build mapping file using the first input file as the grid template + # Construct the mapping file name (symlinked from + # build_mapping_file step) ice_sheet = config.get("ismip7", "ice_sheet") mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" f"{mali_mesh_name}_{method_remap}.nc") - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each year file remapped_files = [] for input_file in input_files: diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py index e35755e9b6..08cc2c971a 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py @@ -7,7 +7,6 @@ from mpas_tools.logging import check_call from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -39,11 +38,22 @@ def setup(self): section = config["ismip7"] base_path_mali = section.get("base_path_mali") mali_mesh_file = section.get("mali_mesh_file") + mali_mesh_name = section.get("mali_mesh_name") + ice_sheet = section.get("ice_sheet") + + section_atm = config["ismip7_atmosphere"] + method_remap = section_atm.get("method_remap") self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) + # Add input file for the mapping file from build_mapping_file step + mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" + f"{mali_mesh_name}_{method_remap}.nc") + self.add_input_file(filename=mapping_file, + target=f"../build_mapping_file/{mapping_file}") + def run(self): """ Run this step of the test case @@ -55,7 +65,6 @@ def run(self): section = config["ismip7"] base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -103,18 +112,12 @@ def run(self): logger.info(f"Found {len(input_files)} SMB gradient files for years " f"{start_year}-{end_year}") - # Build mapping file (reuse if already created by process_smb) + # Construct the mapping file name (symlinked from + # build_mapping_file step) ice_sheet = config.get("ismip7", "ice_sheet") mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" f"{mali_mesh_name}_{method_remap}.nc") - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each year file remapped_files = [] for input_file in input_files: diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py index 56b13403be..441f07d518 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py @@ -7,7 +7,6 @@ from mpas_tools.logging import check_call from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -38,11 +37,22 @@ def setup(self): section = config["ismip7"] base_path_mali = section.get("base_path_mali") mali_mesh_file = section.get("mali_mesh_file") + mali_mesh_name = section.get("mali_mesh_name") + ice_sheet = section.get("ice_sheet") + + section_atm = config["ismip7_atmosphere"] + method_remap = section_atm.get("method_remap") self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) + # Add input file for the mapping file from build_mapping_file step + mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" + f"{mali_mesh_name}_{method_remap}.nc") + self.add_input_file(filename=mapping_file, + target=f"../build_mapping_file/{mapping_file}") + def run(self): """ Run this step of the test case @@ -54,7 +64,6 @@ def run(self): section = config["ismip7"] base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -102,18 +111,12 @@ def run(self): logger.info(f"Found {len(input_files)} temperature files for years " f"{start_year}-{end_year}") - # Build mapping file (reuse if already created by process_smb) + # Construct the mapping file name (symlinked from + # build_mapping_file step) ice_sheet = config.get("ismip7", "ice_sheet") mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" f"{mali_mesh_name}_{method_remap}.nc") - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each year file remapped_files = [] for input_file in input_files: diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py index 41cf9f30aa..add1090a9b 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py @@ -7,7 +7,6 @@ from mpas_tools.logging import check_call from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -40,11 +39,22 @@ def setup(self): section = config["ismip7"] base_path_mali = section.get("base_path_mali") mali_mesh_file = section.get("mali_mesh_file") + mali_mesh_name = section.get("mali_mesh_name") + ice_sheet = section.get("ice_sheet") + + section_atm = config["ismip7_atmosphere"] + method_remap = section_atm.get("method_remap") self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) + # Add input file for the mapping file from build_mapping_file step + mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" + f"{mali_mesh_name}_{method_remap}.nc") + self.add_input_file(filename=mapping_file, + target=f"../build_mapping_file/{mapping_file}") + def run(self): """ Run this step of the test case @@ -56,7 +66,6 @@ def run(self): section = config["ismip7"] base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -104,18 +113,12 @@ def run(self): logger.info(f"Found {len(input_files)} temperature gradient files " f"for years {start_year}-{end_year}") - # Build mapping file (reuse if already created by other steps) + # Construct the mapping file name (symlinked from + # build_mapping_file step) ice_sheet = config.get("ismip7", "ice_sheet") mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" f"{mali_mesh_name}_{method_remap}.nc") - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each year file remapped_files = [] for input_file in input_files: diff --git a/compass/landice/tests/ismip7_forcing/fracture/__init__.py b/compass/landice/tests/ismip7_forcing/fracture/__init__.py index b6a7e63c6a..ba34980e6a 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/__init__.py +++ b/compass/landice/tests/ismip7_forcing/fracture/__init__.py @@ -1,6 +1,9 @@ from compass.landice.tests.ismip7_forcing.configure import ( configure as configure_testgroup, ) +from compass.landice.tests.ismip7_forcing.fracture.build_mapping_file import ( + BuildMappingFile, +) from compass.landice.tests.ismip7_forcing.fracture.process_excess_melt import ( ProcessExcessMelt, ) @@ -40,6 +43,7 @@ def __init__(self, test_group): subdir = name super().__init__(test_group=test_group, name=name, subdir=subdir) + self.add_step(BuildMappingFile(test_case=self)) self.add_step(ProcessExcessMelt(test_case=self)) self.add_step(ProcessLakeProperties(test_case=self)) self.add_step(ProcessShelfCollapse(test_case=self)) diff --git a/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py new file mode 100644 index 0000000000..17a68ac529 --- /dev/null +++ b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py @@ -0,0 +1,142 @@ +import glob +import os +import shutil + +from compass.landice.ismip7.mapping import build_mapping_file +from compass.step import Step + + +class BuildMappingFile(Step): + """ + A step for building ESMF mapping files (regridding weights) from the + ISMIP7 fracture grid to the MALI mesh. This step runs ESMF_RegridWeightGen + with the full esmf_ntasks allocation, so it can be run on a separate node + allocation from the processing steps (which run on a single node). + + Up to three mapping files may be built, one for each enabled fracture + pathway (shelf collapse, excess melt, lake properties), since each may + use a different remapping method. The mapping files can be reused across + scenarios. If mapping_files_path is provided in the config, this step will + symlink any existing weight files from that directory and skip building + those that already exist. + """ + + def __init__(self, test_case): + """ + Create the step + + Parameters + ---------- + test_case : compass.landice.tests.ismip7_forcing.fracture.Fracture + The test case this step belongs to + """ + super().__init__(test_case=test_case, name="build_mapping_file") + + def setup(self): + """ + Set up this step of the test case + """ + config = self.config + section = config["ismip7"] + base_path_mali = section.get("base_path_mali") + mali_mesh_file = section.get("mali_mesh_file") + + self.add_input_file(filename=mali_mesh_file, + target=os.path.join(base_path_mali, + mali_mesh_file)) + + # Request the full esmf_ntasks allocation for this step + self.ntasks = section.getint("esmf_ntasks") + self.min_tasks = self.ntasks + + def run(self): + """ + Run this step of the test case + """ + logger = self.logger + config = self.config + + section = config["ismip7"] + base_path_ismip7 = section.get("base_path_ismip7") + mali_mesh_name = section.get("mali_mesh_name") + mali_mesh_file = section.get("mali_mesh_file") + ice_sheet = section.get("ice_sheet") + output_base_path = section.get("output_base_path") + + # Check if user provided a path to reuse existing weight files + mapping_files_path = section.get("mapping_files_path") + + section = config["ismip7_fracture"] + version = section.get("version") + + # Three fracture pathways, each with its own remapping method + fracture_paths = [ + ("shelf_collapse", "method_remap_shelf_collapse", + "ice_shelf_collapse_mask_*.nc"), + ("excess_melt", "method_remap_excess_melt", + "excess_meltwaterinput_*.nc"), + ("lake_properties", "method_remap_lake_properties", + "lake_properties_*.nc"), + ] + + # Build a mapping file for each enabled pathway + for pathway_name, method_key, file_pattern in fracture_paths: + method_remap = section.get(method_key) + + # Skip if method is None + if method_remap.lower() == "none": + logger.info(f"{method_key} is None; skipping {pathway_name} " + f"mapping file.") + continue + + # Construct the canonical mapping file name + mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" + f"{mali_mesh_name}_{method_remap}.nc") + + # If mapping_files_path is provided, symlink any existing file + if mapping_files_path != "NotAvailable": + source_file = os.path.join(mapping_files_path, mapping_file) + if os.path.exists(source_file): + logger.info(f"Symlinking existing mapping file for " + f"{pathway_name} from {mapping_files_path}") + if os.path.exists(mapping_file): + os.remove(mapping_file) + os.symlink(source_file, mapping_file) + + # Use the pathway's fracture file as the grid template + input_path = os.path.join(base_path_ismip7, "fracture", version) + grid_files = sorted( + glob.glob(os.path.join(input_path, file_pattern))) + if not grid_files: + raise FileNotFoundError( + f"No {pathway_name} file found matching pattern:\n" + f" {os.path.join(input_path, file_pattern)}") + + ismip7_grid_file = grid_files[0] + logger.info(f"Building {pathway_name} mapping file using grid " + f"template: {os.path.basename(ismip7_grid_file)}") + + # Build the mapping file (build_mapping_file will skip if it + # already exists, e.g., from the symlink above or a previous + # failed run) + build_mapping_file(config, logger, ismip7_grid_file, mapping_file, + mali_mesh_file=mali_mesh_file, + method_remap=method_remap) + + # Copy the mapping file to output_base_path/mapping_files/ for + # reuse in future runs + mapping_files_dir = os.path.join(output_base_path, "mapping_files") + if not os.path.exists(mapping_files_dir): + os.makedirs(mapping_files_dir) + + dst = os.path.join(mapping_files_dir, mapping_file) + # Only copy if it's a real file (not a symlink we just created) + if not os.path.islink(mapping_file): + logger.info(f"Copying {pathway_name} mapping file to " + f"{mapping_files_dir} for reuse") + shutil.copy(mapping_file, dst) + else: + logger.info(f"{pathway_name} mapping file is a symlink; not " + f"copying to {mapping_files_dir}") + + logger.info("Done building fracture mapping files.") diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py index 0f291792f0..2c432158bc 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py @@ -7,7 +7,6 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import ( add_xtime_and_write, extrapolate_source, @@ -49,11 +48,24 @@ def setup(self): section = config["ismip7"] base_path_mali = section.get("base_path_mali") mali_mesh_file = section.get("mali_mesh_file") + mali_mesh_name = section.get("mali_mesh_name") + ice_sheet = section.get("ice_sheet") + + section_frac = config["ismip7_fracture"] + method_remap = section_frac.get("method_remap_excess_melt") self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) + # Add input file for the mapping file from build_mapping_file step + # (only if method is not None) + if method_remap.lower() != "none": + mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" + f"{mali_mesh_name}_{method_remap}.nc") + self.add_input_file(filename=mapping_file, + target=f"../build_mapping_file/{mapping_file}") + def run(self): """ Run this step of the test case @@ -64,7 +76,6 @@ def run(self): section = config["ismip7"] base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -105,18 +116,10 @@ def run(self): self._prepare_source_grid(input_file, input_path, gridded_file, logger) - # Build mapping file. Excess melt is a flux, so conservative - # remapping is appropriate by default. + # Construct mapping file name (symlinked from build_mapping_file step) mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" f"{mali_mesh_name}_{method_remap}.nc") - if not os.path.exists(mapping_file): - logger.info("Building mapping file for the excess melt grid...") - build_mapping_file(config, logger, - gridded_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Extrapolate fill values on the source grid before remapping so # they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py index 2b5d5ac1d5..e6cc092abf 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py @@ -4,7 +4,6 @@ from mpas_tools.logging import check_call -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import ( add_xtime_and_write, extrapolate_source, @@ -54,11 +53,24 @@ def setup(self): section = config["ismip7"] base_path_mali = section.get("base_path_mali") mali_mesh_file = section.get("mali_mesh_file") + mali_mesh_name = section.get("mali_mesh_name") + ice_sheet = section.get("ice_sheet") + + section_frac = config["ismip7_fracture"] + method_remap = section_frac.get("method_remap_lake_properties") self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) + # Add input file for the mapping file from build_mapping_file step + # (only if method is not None) + if method_remap.lower() != "none": + mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" + f"{mali_mesh_name}_{method_remap}.nc") + self.add_input_file(filename=mapping_file, + target=f"../build_mapping_file/{mapping_file}") + def run(self): """ Run this step of the test case @@ -69,7 +81,6 @@ def run(self): section = config["ismip7"] base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -105,19 +116,10 @@ def run(self): basename = os.path.basename(input_file) logger.info(f"Processing lake properties: {basename}") - # Build mapping file. Lake properties are continuous fields, so - # bilinear remapping is appropriate by default. + # Construct mapping file name (symlinked from build_mapping_file step) mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" f"{mali_mesh_name}_{method_remap}.nc") - if not os.path.exists(mapping_file): - logger.info("Building mapping file for the lake properties " - "grid...") - build_mapping_file(config, logger, - input_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Extrapolate fill values on the source grid before remapping so # they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py index bf6537438e..f4c0b67ba9 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py @@ -4,7 +4,6 @@ from mpas_tools.logging import check_call -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import ( add_xtime_and_write, open_rename_and_trim, @@ -41,11 +40,24 @@ def setup(self): section = config["ismip7"] base_path_mali = section.get("base_path_mali") mali_mesh_file = section.get("mali_mesh_file") + mali_mesh_name = section.get("mali_mesh_name") + ice_sheet = section.get("ice_sheet") + + section_frac = config["ismip7_fracture"] + method_remap = section_frac.get("method_remap_shelf_collapse") self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) + # Add input file for the mapping file from build_mapping_file step + # (only if method is not None) + if method_remap.lower() != "none": + mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" + f"{mali_mesh_name}_{method_remap}.nc") + self.add_input_file(filename=mapping_file, + target=f"../build_mapping_file/{mapping_file}") + def run(self): """ Run this step of the test case @@ -56,7 +68,6 @@ def run(self): section = config["ismip7"] base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -92,17 +103,10 @@ def run(self): basename = os.path.basename(input_file) logger.info(f"Processing ice shelf collapse mask: {basename}") - # Build mapping file. neareststod preserves the 0/1 mask values. + # Construct mapping file name (symlinked from build_mapping_file step) mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" f"{mali_mesh_name}_{method_remap}.nc") - if not os.path.exists(mapping_file): - logger.info("Building mapping file for the collapse mask grid...") - build_mapping_file(config, logger, - input_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap the collapse mask onto the MALI mesh remapped_file = f"remapped_{basename}" if not os.path.exists(remapped_file): diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg index be7d195aa4..e2104c4f6a 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg @@ -30,6 +30,13 @@ mali_mesh_file = NotAvailable # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 128 +# Optional path to a directory of pre-built mapping files for reuse across runs. +# If provided, the test case will symlink any existing, canonically-named weight +# files from this directory and build only the missing ones. To reuse weights from +# a previous run, point this at /mapping_files/. +# If not provided or set to NotAvailable, all weight files are built from scratch. +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg index 975cb89b85..43d4efaa52 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg @@ -39,6 +39,13 @@ mali_mesh_file = AIS_4to20km_r01_20220907.nc # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 512 +# Optional path to a directory of pre-built mapping files for reuse across runs. +# If provided, the test case will symlink any existing, canonically-named weight +# files from this directory and build only the missing ones. To reuse weights from +# a previous run, point this at /mapping_files/. +# If not provided or set to NotAvailable, all weight files are built from scratch. +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg index d3658027db..57df8aa2dd 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg @@ -37,6 +37,13 @@ mali_mesh_file = GIS_1to10km_r02_20230202.nc # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 512 +# Optional path to a directory of pre-built mapping files for reuse across runs. +# If provided, the test case will symlink any existing, canonically-named weight +# files from this directory and build only the missing ones. To reuse weights from +# a previous run, point this at /mapping_files/. +# If not provided or set to NotAvailable, all weight files are built from scratch. +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg index ab282b2e06..b5c795bf3d 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg @@ -30,6 +30,13 @@ mali_mesh_file = AIS_4to20km_r01_20220907.nc # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 512 +# Optional path to a directory of pre-built mapping files for reuse across runs. +# If provided, the test case will symlink any existing, canonically-named weight +# files from this directory and build only the missing ones. To reuse weights from +# a previous run, point this at /mapping_files/. +# If not provided or set to NotAvailable, all weight files are built from scratch. +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = false diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py index 2a97700428..e384db2cb4 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py @@ -1,6 +1,9 @@ from compass.landice.tests.ismip7_forcing.configure import ( configure as configure_testgroup, ) +from compass.landice.tests.ismip7_forcing.ocean_thermal.build_mapping_file import ( # noqa: E501 + BuildMappingFile, +) from compass.landice.tests.ismip7_forcing.ocean_thermal.process_thermal_forcing import ( # noqa: E501 ProcessThermalForcing, ) @@ -29,6 +32,7 @@ def __init__(self, test_group): subdir = name super().__init__(test_group=test_group, name=name, subdir=subdir) + self.add_step(BuildMappingFile(test_case=self)) self.add_step(ProcessThermalForcing(test_case=self)) def configure(self): diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py new file mode 100644 index 0000000000..9fd3812389 --- /dev/null +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py @@ -0,0 +1,152 @@ +import glob +import os +import shutil + +from compass.landice.ismip7.ice_sheet_params import get_params +from compass.landice.ismip7.mapping import build_mapping_file +from compass.step import Step + + +class BuildMappingFile(Step): + """ + A step for building the ESMF mapping file (regridding weights) from the + ISMIP7 ocean grid to the MALI mesh. This step runs ESMF_RegridWeightGen + with the full esmf_ntasks allocation, so it can be run on a separate node + allocation from the processing steps (which run on a single node). + + The mapping file can be reused across scenarios and ocean choices, since + it depends only on the ice sheet, mesh name, and remapping method. If + mapping_files_path is provided in the config, this step will symlink any + existing weight file from that directory and skip the build. + """ + + def __init__(self, test_case): + """ + Create the step + + Parameters + ---------- + test_case : compass.landice.tests.ismip7_forcing.ocean_thermal.\ +OceanThermal + The test case this step belongs to + """ + super().__init__(test_case=test_case, name="build_mapping_file") + + def setup(self): + """ + Set up this step of the test case + """ + config = self.config + section = config["ismip7"] + base_path_mali = section.get("base_path_mali") + mali_mesh_file = section.get("mali_mesh_file") + + self.add_input_file(filename=mali_mesh_file, + target=os.path.join(base_path_mali, + mali_mesh_file)) + + # Request the full esmf_ntasks allocation for this step + self.ntasks = section.getint("esmf_ntasks") + self.min_tasks = self.ntasks + + def run(self): + """ + Run this step of the test case + """ + logger = self.logger + config = self.config + params = get_params(config) + + section = config["ismip7"] + base_path_ismip7 = section.get("base_path_ismip7") + mali_mesh_name = section.get("mali_mesh_name") + mali_mesh_file = section.get("mali_mesh_file") + ice_sheet = section.get("ice_sheet") + output_base_path = section.get("output_base_path") + model = section.get("model") + scenario = section.get("scenario") + + # Check if user provided a path to reuse existing weight files + mapping_files_path = section.get("mapping_files_path") + + section = config["ismip7_ocean_thermal"] + method_remap = section.get("method_remap") + + # Construct the canonical mapping file name + mapping_file = (f"map_ismip7_{ice_sheet}_ocean_to_" + f"{mali_mesh_name}_{method_remap}.nc") + + # If mapping_files_path is provided, symlink any existing file + if mapping_files_path != "NotAvailable": + source_file = os.path.join(mapping_files_path, mapping_file) + if os.path.exists(source_file): + logger.info(f"Symlinking existing mapping file from " + f"{mapping_files_path}") + if os.path.exists(mapping_file): + os.remove(mapping_file) + os.symlink(source_file, mapping_file) + + # Use any ocean thermal forcing file as the grid template for building + # the mapping file. Look for the first available thermal_forcing file. + prefix = params['prefix'] + version = params['ocean_version'] + ocean_grid = params['ocean_grid'] + + # For OCX scenarios, the directory structure differs + if params.get('ocean_choice_layout', False): + # AIS OCX: files are in subdirectories (main/cold/warm/vary) + # Use 'main' as the default choice for building the mapping file + ocean_choice = 'main' + input_path = os.path.join( + base_path_ismip7, "thermal_forcing", version, ocean_choice) + file_pattern = (f"thermal_forcing_{prefix}_{ocean_grid}_" + f"{version}_*.nc") + else: + # Standard ESM scenario or GrIS OCX + if params['ocean_model'] is not None: + # OCX (GrIS): use the ocean_model token + ocean_model = params['ocean_model'] + input_path = os.path.join( + base_path_ismip7, "thermal_forcing", version) + file_pattern = (f"thermal_forcing_{prefix}_{ocean_model}_" + f"{ocean_grid}_{version}_*.nc") + else: + # Standard ESM scenario + input_path = os.path.join( + base_path_ismip7, "thermal_forcing", version) + file_pattern = (f"thermal_forcing_{prefix}_{model}_" + f"{scenario}_{ocean_grid}_{version}_*.nc") + + ismip7_grid_files = sorted( + glob.glob(os.path.join(input_path, file_pattern))) + if not ismip7_grid_files: + raise FileNotFoundError( + f"No ocean thermal forcing file found matching pattern:\n" + f" {os.path.join(input_path, file_pattern)}") + + ismip7_grid_file = ismip7_grid_files[0] + logger.info(f"Using grid template: " + f"{os.path.basename(ismip7_grid_file)}") + + # Build the mapping file (build_mapping_file will skip if it already + # exists, e.g., from the symlink above or a previous failed run) + build_mapping_file(config, logger, ismip7_grid_file, mapping_file, + mali_mesh_file=mali_mesh_file, + method_remap=method_remap) + + # Copy the mapping file to output_base_path/mapping_files/ for reuse + # in future runs + mapping_files_dir = os.path.join(output_base_path, "mapping_files") + if not os.path.exists(mapping_files_dir): + os.makedirs(mapping_files_dir) + + dst = os.path.join(mapping_files_dir, mapping_file) + # Only copy if it's a real file (not a symlink we just created) + if not os.path.islink(mapping_file): + logger.info(f"Copying mapping file to {mapping_files_dir} for reuse") + shutil.copy(mapping_file, dst) + else: + logger.info(f"Mapping file is a symlink; not copying to " + f"{mapping_files_dir}") + + logger.info("Done building ocean mapping file.") diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py index 595d90b964..9870ea5b18 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py @@ -7,7 +7,6 @@ from mpas_tools.logging import check_call from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -42,11 +41,22 @@ def setup(self): section = config["ismip7"] base_path_mali = section.get("base_path_mali") mali_mesh_file = section.get("mali_mesh_file") + mali_mesh_name = section.get("mali_mesh_name") + ice_sheet = section.get("ice_sheet") + + section_ocean = config["ismip7_ocean_thermal"] + method_remap = section_ocean.get("method_remap") self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) + # Add input file for the mapping file from build_mapping_file step + mapping_file = (f"map_ismip7_{ice_sheet}_ocean_to_" + f"{mali_mesh_name}_{method_remap}.nc") + self.add_input_file(filename=mapping_file, + target=f"../build_mapping_file/{mapping_file}") + def run(self): """ Run this step of the test case @@ -249,7 +259,6 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, Base path under which output is written """ logger = self.logger - config = self.config input_path = job['input_path'] file_pattern = job['file_pattern'] @@ -284,13 +293,7 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, logger.info(f"Found {len(input_files)} ocean thermal forcing files " f"overlapping years {start_year}-{end_year}") - # Build mapping file using the first input file as grid template. - if not os.path.exists(mapping_file): - logger.info("Building mapping file for ocean grid...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) + # Mapping file is symlinked from build_mapping_file step # Remap each file remapped_files = [] @@ -385,17 +388,10 @@ def _run_climatology(self): logger.info(f"Processing ocean TF climatology: " f"{os.path.basename(input_file)}") - # Build mapping file using the climatology file as grid template. + # Construct mapping file name (symlinked from build_mapping_file step) mapping_file = (f"map_ismip7_{ice_sheet}_ocean_to_" f"{mali_mesh_name}_{method_remap}.nc") - if not os.path.exists(mapping_file): - logger.info("Building mapping file for ocean grid...") - build_mapping_file(config, logger, - input_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Extrapolate and remap basename = os.path.basename(input_file) remapped_file = f"remapped_{basename}" From 333849add697cf2efc12a645cfe88d2b790e6601 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Tue, 22 Sep 2026 13:27:47 -0600 Subject: [PATCH 2/3] Document BuildMappingFile steps and weight reuse in ISMIP7 forcing Update both user and developer guides to document the new BuildMappingFile steps and workflow optimization strategies for the ISMIP7 forcing test group. User's Guide changes: - Update test case descriptions to include build_mapping_file as first step - Add new "Workflow Optimization" section explaining resource allocation strategies (separate allocations for weight-gen vs processing) - Document weight reuse across scenarios via mapping_files_path config option - Add mapping_files_path to config options documentation - Update atmosphere, ocean_thermal, and fracture sections to describe the BuildMappingFile steps Developer's Guide changes: - Update framework section to clarify build_mapping_file is now called only from BuildMappingFile steps, not from processing steps - Document BuildMappingFile classes for atmosphere, ocean_thermal, and fracture test cases - Explain architectural separation: weight-generation runs at esmf_ntasks, processing steps run at ntasks=1 - Update step listings and descriptions for all three test cases Key documentation points: - ESMF_RegridWeightGen runs for 5-10 minutes on 128-512 tasks - Processing steps run for hours on single nodes - Mapping files depend only on ice_sheet + mali_mesh_name + method_remap - Weights built for one scenario can be reused for other scenarios with the same mesh and remapping method Co-Authored-By: Claude Sonnet 4.5 --- .../atmosphere/build_mapping_file.py | 3 +- .../ocean_thermal/build_mapping_file.py | 3 +- .../ocean_thermal/process_thermal_forcing.py | 1 - .../landice/test_groups/ismip7_forcing.rst | 99 ++++++++++---- .../landice/test_groups/ismip7_forcing.rst | 128 +++++++++++++++--- 5 files changed, 187 insertions(+), 47 deletions(-) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py index fd1df2c948..73520b3aa7 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py @@ -123,7 +123,8 @@ def run(self): dst = os.path.join(mapping_files_dir, mapping_file) # Only copy if it's a real file (not a symlink we just created) if not os.path.islink(mapping_file): - logger.info(f"Copying mapping file to {mapping_files_dir} for reuse") + logger.info(f"Copying mapping file to {mapping_files_dir} " + f"for reuse") shutil.copy(mapping_file, dst) else: logger.info(f"Mapping file is a symlink; not copying to " diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py index 9fd3812389..cc09fed192 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py @@ -143,7 +143,8 @@ def run(self): dst = os.path.join(mapping_files_dir, mapping_file) # Only copy if it's a real file (not a symlink we just created) if not os.path.islink(mapping_file): - logger.info(f"Copying mapping file to {mapping_files_dir} for reuse") + logger.info(f"Copying mapping file to {mapping_files_dir} " + f"for reuse") shutil.copy(mapping_file, dst) else: logger.info(f"Mapping file is a symlink; not copying to " diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py index 9870ea5b18..cac23903cf 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py @@ -365,7 +365,6 @@ def _run_climatology(self): section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") output_base_path = section.get("output_base_path") ice_sheet = section.get("ice_sheet") diff --git a/docs/developers_guide/landice/test_groups/ismip7_forcing.rst b/docs/developers_guide/landice/test_groups/ismip7_forcing.rst index 6f88904b99..fa3b6c2e5b 100644 --- a/docs/developers_guide/landice/test_groups/ismip7_forcing.rst +++ b/docs/developers_guide/landice/test_groups/ismip7_forcing.rst @@ -26,7 +26,8 @@ package :py:mod:`compass.landice.ismip7`, described in ice-sheet-specific parameters (projection, file naming prefix, grid resolution, data version, ocean dimensionality), :py:func:`compass.landice.ismip7.mapping.build_mapping_file` to create the -SCRIP and ESMF mapping files, and the remapping helpers in +SCRIP and ESMF mapping files (called from ``BuildMappingFile`` steps, not from +processing steps), and the remapping helpers in :py:mod:`compass.landice.ismip7.remap`. When ``scenario = OCX``, ``get_params`` applies a set of OCX overrides on top @@ -61,14 +62,28 @@ atmosphere ~~~~~~~~~~ The :py:class:`compass.landice.tests.ismip7_forcing.atmosphere.Atmosphere` -test case processes the ISMIP7 atmosphere forcing fields. It contains five -steps: SMB, temperature, their respective gradients, and runoff. Each step -discovers input files matching the ice-sheet-specific naming pattern, builds -or reuses a mapping file, remaps each input file with ``ncremap``, and -combines/renames the results to MALI conventions. +test case processes the ISMIP7 atmosphere forcing fields. It contains six +steps: a weight-generation step followed by five processing steps (SMB, +temperature, their respective gradients, and runoff). + +The weight-generation step runs ``ESMF_RegridWeightGen`` with ``esmf_ntasks`` +MPI tasks to build regridding weights from the ISMIP7 atmosphere grid to the +MALI mesh. All five processing steps consume the same mapping file (all +atmosphere variables share the same source grid). + +Each processing step discovers input files matching the ice-sheet-specific +naming pattern, remaps each input file with ``ncremap`` using the pre-built +mapping file, and combines/renames the results to MALI conventions. Processing +steps run at ``ntasks=1`` (single node). Steps: +* :py:class:`~compass.landice.tests.ismip7_forcing.atmosphere.build_mapping_file.BuildMappingFile` — + Builds one shared mapping file for all atmosphere variables. The file is + copied to ``/mapping_files/`` for reuse across scenarios. + If ``mapping_files_path`` is provided in the config, symlinks any existing + weight file from that directory and skips the build. + * :py:class:`~compass.landice.tests.ismip7_forcing.atmosphere.process_smb.ProcessSmb` — ``acabf`` → ``sfcMassBal`` * :py:class:`~compass.landice.tests.ismip7_forcing.atmosphere.process_temperature.ProcessTemperature` — @@ -86,20 +101,39 @@ ocean_thermal ~~~~~~~~~~~~~ The :py:class:`compass.landice.tests.ismip7_forcing.ocean_thermal.OceanThermal` -test case processes the ISMIP7 ocean thermal forcing. It contains a single step, +test case processes the ISMIP7 ocean thermal forcing. It contains two steps: a +weight-generation step followed by the processing step. + +The weight-generation step runs ``ESMF_RegridWeightGen`` with ``esmf_ntasks`` +MPI tasks to build regridding weights from the ISMIP7 ocean grid to the MALI +mesh. One mapping file is built and shared across scenario and climatology +processing (if both are enabled and use the same remapping method). + +The processing step, :py:class:`~compass.landice.tests.ismip7_forcing.ocean_thermal.process_thermal_forcing.ProcessThermalForcing`, -which handles both AIS (3D, decade-spanning files) and GrIS (2D, yearly files) -by branching on the ``ocean_3d`` parameter from ``ice_sheet_params``. +handles both AIS (3D, decade-spanning files) and GrIS (2D, yearly files) by +branching on the ``ocean_3d`` parameter from ``ice_sheet_params``. It runs at +``ntasks=1`` (single node). -The ``run()`` method dispatches to two sub-methods based on the boolean config -options ``process_ocean_thermal`` and ``process_ocean_climatology`` in the -``[ismip7]`` section: +Steps: + +* :py:class:`~compass.landice.tests.ismip7_forcing.ocean_thermal.build_mapping_file.BuildMappingFile` — + Builds one mapping file for ocean thermal forcing (shared across scenario and + climatology if methods match). The file is copied to + ``/mapping_files/`` for reuse across scenarios. If + ``mapping_files_path`` is provided in the config, symlinks any existing + weight file from that directory and skips the build. -* ``_run_scenario()``: Processes time-varying ESM scenario data (model + - scenario combination). Uses config from ``[ismip7_ocean_thermal]``. -* ``_run_climatology()``: Processes the static observational climatology - (Zhou et al., AIS only). Uses config from ``[ismip7_ocean_climatology]``. - The TF version (currently v3) is hard-coded. +* :py:class:`~compass.landice.tests.ismip7_forcing.ocean_thermal.process_thermal_forcing.ProcessThermalForcing` — + The ``run()`` method dispatches to two sub-methods based on the boolean + config options ``process_ocean_thermal`` and ``process_ocean_climatology`` in + the ``[ismip7]`` section: + + * ``_run_scenario()``: Processes time-varying ESM scenario data (model + + scenario combination). Uses config from ``[ismip7_ocean_thermal]``. + * ``_run_climatology()``: Processes the static observational climatology + (Zhou et al., AIS only). Uses config from ``[ismip7_ocean_climatology]``. + The TF version (currently v3) is hard-coded. For AIS scenario data, the step: @@ -127,17 +161,32 @@ fracture The :py:class:`compass.landice.tests.ismip7_forcing.fracture.Fracture` test case processes the ISMIP7 surface-melt-driven ice shelf collapse -forcing (AIS only). It implements the three ISMIP7 pathways as independent -steps, each discovering its source file from the ``fracture/{version}/`` -subdirectory of ``base_path_ismip7``, building or reusing a mapping file, -remapping with ``ncremap``, and renaming the result to MALI conventions with -an accompanying ``xtime`` variable. Per-pathway remapping methods are set in -the ``[ismip7_fracture]`` config section. Setting a pathway's remapping-method -option to ``None`` causes that step to return early without processing its -file, which is useful when only some pathway source files are available. +forcing (AIS only). It contains four steps: a weight-generation step followed +by three pathway processing steps. + +The weight-generation step runs ``ESMF_RegridWeightGen`` with ``esmf_ntasks`` +MPI tasks to build regridding weights from the ISMIP7 fracture grid to the +MALI mesh. Up to three mapping files may be built (one per enabled pathway), +depending on whether the pathways use different remapping methods. + +Each processing step discovers its source file from the ``fracture/{version}/`` +subdirectory of ``base_path_ismip7``, remaps with ``ncremap`` using the +pre-built mapping file, and renames the result to MALI conventions with an +accompanying ``xtime`` variable. Processing steps run at ``ntasks=1`` (single +node). Per-pathway remapping methods are set in the ``[ismip7_fracture]`` +config section. Setting a pathway's remapping-method option to ``None`` causes +that step to return early without processing its file, which is useful when +only some pathway source files are available. Steps: +* :py:class:`~compass.landice.tests.ismip7_forcing.fracture.build_mapping_file.BuildMappingFile` — + Builds up to three mapping files (one per enabled pathway with a non-None + remapping method). Files are copied to ``/mapping_files/`` + for reuse across scenarios. If ``mapping_files_path`` is provided in the + config, symlinks any existing weight files from that directory and builds + only the missing ones. + * :py:class:`~compass.landice.tests.ismip7_forcing.fracture.process_excess_melt.ProcessExcessMelt` (Path A) — ``excess_melt`` → ``ismip7ExcessMelt``. The excess melt file lacks ``x``/``y`` coordinate variables and its array is flipped along the diff --git a/docs/users_guide/landice/test_groups/ismip7_forcing.rst b/docs/users_guide/landice/test_groups/ismip7_forcing.rst index 4a5ca9ea82..8e800e8862 100644 --- a/docs/users_guide/landice/test_groups/ismip7_forcing.rst +++ b/docs/users_guide/landice/test_groups/ismip7_forcing.rst @@ -11,20 +11,23 @@ The test group supports both the Antarctic Ice Sheet (AIS) and the Greenland Ice Sheet (GrIS), controlled by a single ``ice_sheet`` config option. The test group includes three test cases: ``atmosphere``, ``ocean_thermal``, -and ``fracture``. +and ``fracture``. Each test case begins with a ``build_mapping_file`` step that +generates ESMF regridding weights, followed by processing steps that consume +those weights to remap forcing data. -* The ``atmosphere`` test case has five steps: +* The ``atmosphere`` test case has six steps: ``build_mapping_file``, ``process_smb``, ``process_temperature``, ``process_smb_gradient``, ``process_temperature_gradient``, and ``process_runoff``. -* The ``ocean_thermal`` test case has one step: ``process_thermal_forcing``. - For AIS this produces 3D thermal forcing (with 30 ocean depth layers); for - GrIS it produces 2D (depth-averaged) thermal forcing. The step can also - process the observational ocean thermal forcing climatology (Zhou et al.) - for AIS, controlled by the ``process_ocean_climatology`` config option. +* The ``ocean_thermal`` test case has two steps: ``build_mapping_file`` and + ``process_thermal_forcing``. For AIS this produces 3D thermal forcing (with + 30 ocean depth layers); for GrIS it produces 2D (depth-averaged) thermal + forcing. The processing step can also handle the observational ocean thermal + forcing climatology (Zhou et al.) for AIS, controlled by the + ``process_ocean_climatology`` config option. -* The ``fracture`` test case has three steps: ``process_excess_melt`` - (Path A), ``process_lake_properties`` (Path B), and +* The ``fracture`` test case has four steps: ``build_mapping_file``, + ``process_excess_melt`` (Path A), ``process_lake_properties`` (Path B), and ``process_shelf_collapse`` (Path C). It processes the ISMIP7 surface-melt-driven ice shelf collapse forcing (AIS only). @@ -54,6 +57,58 @@ To use this test group, users need to: combination to process the surface-melt-driven ice shelf collapse pathways (excess melt, lake properties, and the ice shelf collapse mask). +.. _landice_ismip7_forcing_workflow_optimization: + +Workflow Optimization +~~~~~~~~~~~~~~~~~~~~~ + +Each test case separates weight-generation (``build_mapping_file`` step) from +processing (``process_*`` steps) to enable efficient resource allocation. +``ESMF_RegridWeightGen`` benefits from many MPI tasks (128-512) but runs for +only 5-10 minutes, while the processing steps run efficiently on a single node +but may take hours to remap dozens of yearly files. + +**Separate allocation strategy**: Run ``build_mapping_file`` on a large +allocation, then run the processing steps on a smaller allocation: + +.. code-block:: bash + + # Allocate 128+ tasks for weight generation (5-10 minutes) + compass run -s build_mapping_file + + # Allocate 1 task for processing (hours) + compass run -s process_smb process_temperature process_smb_gradient \ + process_temperature_gradient process_runoff + +This avoids wasting hundreds of node-hours by holding a large allocation during +the long processing phase. + +**Weight reuse across scenarios**: Mapping files depend only on ``ice_sheet``, +``mali_mesh_name``, and ``method_remap``—not on model or scenario. This means +weights built for ssp585 are identical to those for ssp126 (same mesh and +method) and can be reused via the ``mapping_files_path`` config option: + +.. code-block:: cfg + + [ismip7] + # Point to a previous run's mapping_files directory + mapping_files_path = /path/to/output_base_path/mapping_files/ + +When ``mapping_files_path`` is set, ``build_mapping_file`` symlinks any existing +weight files from that directory and builds only the missing ones. Built weights +are automatically copied to ``/mapping_files/`` for reuse in +future runs. + +**Typical multi-scenario workflow**: + +1. Process the first scenario (e.g., ssp126) normally. Built weights are saved + to ``/mapping_files/``. + +2. For subsequent scenarios with the same mesh and methods, set + ``mapping_files_path`` to reuse the weights. The ``build_mapping_file`` step + completes instantly (just symlinking), and you can skip the large allocation + entirely. + Example user config files are provided in the source tree for local testing: * ``compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg`` @@ -211,6 +266,13 @@ values are: # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 128 + # Optional path to a directory of pre-built mapping files for reuse across runs. + # If provided, the test case will symlink any existing, canonically-named weight + # files from this directory and build only the missing ones. To reuse weights from + # a previous run, point this at /mapping_files/. + # If not provided or set to NotAvailable, all weight files are built from scratch. + mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true @@ -294,6 +356,13 @@ grid to the MALI unstructured mesh. Steps: +* **build_mapping_file**: Builds the ESMF regridding weights from the ISMIP7 + atmosphere grid to the MALI mesh. This step runs ``ESMF_RegridWeightGen`` + with ``esmf_ntasks`` MPI tasks. One mapping file is built and shared across + all five atmosphere variables (they use the same source grid). Built weights + are copied to ``/mapping_files/`` for reuse across + scenarios. + * **process_smb**: Remaps the surface mass balance (``acabf``) field. The output variable is ``sfcMassBal``. @@ -320,17 +389,27 @@ The ``landice/ismip7_forcing/ocean_thermal`` test case processes the ISMIP7 ocean thermal forcing (``tf``) and remaps it from the native polar stereographic grid to the MALI unstructured mesh. -The step supports two processing modes, controlled by boolean config options -in the ``[ismip7]`` section: +Steps: + +* **build_mapping_file**: Builds the ESMF regridding weights from the ISMIP7 + ocean grid to the MALI mesh. This step runs ``ESMF_RegridWeightGen`` with + ``esmf_ntasks`` MPI tasks. One mapping file is built and shared across + scenario and climatology processing (if both are enabled and use the same + remapping method). Built weights are copied to + ``/mapping_files/`` for reuse across scenarios. + +* **process_thermal_forcing**: Processes the ocean thermal forcing data. This + step supports two processing modes, controlled by boolean config options in + the ``[ismip7]`` section: -* **Scenario (time-varying) data** (``process_ocean_thermal = true``): - Processes ESM-driven thermal forcing for a given model/scenario combination. + * **Scenario (time-varying) data** (``process_ocean_thermal = true``): + Processes ESM-driven thermal forcing for a given model/scenario combination. -* **Observational climatology** (``process_ocean_climatology = true``): - Processes the static Zhou et al. observational thermal forcing climatology - (AIS only). This is a time-invariant 3D field referenced to 1995-2024. + * **Observational climatology** (``process_ocean_climatology = true``): + Processes the static Zhou et al. observational thermal forcing climatology + (AIS only). This is a time-invariant 3D field referenced to 1995-2024. -Both modes can be enabled simultaneously. + Both modes can be enabled simultaneously. For **AIS** scenario data, thermal forcing is 3D with 30 vertical ocean layers. The input files span decades (e.g., 1850-1859). The output variable @@ -353,8 +432,9 @@ fracture The ``landice/ismip7_forcing/fracture`` test case processes the ISMIP7 surface-melt-driven ice shelf collapse forcing (AIS only). It implements the -three ISMIP7 pathways as separate steps, each remapping annual fields from -the native 8km polar stereographic grid onto the MALI unstructured mesh. +three ISMIP7 pathways as separate processing steps, each remapping annual +fields from the native 8km polar stereographic grid onto the MALI unstructured +mesh. All three source files are discovered from the ``fracture/{version}/`` subdirectory of ``base_path_ismip7``. @@ -364,6 +444,16 @@ remapping-method config option to ``None`` in the ``[ismip7_fracture]`` section (for example, ``method_remap_excess_melt = None`` skips Path A). This is useful when only some of the pathway source files are available. +Steps: + +* **build_mapping_file**: Builds ESMF regridding weights from the ISMIP7 + fracture grid to the MALI mesh. This step runs ``ESMF_RegridWeightGen`` with + ``esmf_ntasks`` MPI tasks. Up to three mapping files may be built (one per + enabled pathway), depending on whether the pathways use different remapping + methods. If all three pathways use the same method, one mapping file is built + and shared. Built weights are copied to ``/mapping_files/`` + for reuse across scenarios. + * **process_excess_melt** (Path A): Remaps the excess meltwater field (melt + rain after firn air content depletion), matching ``excess_melt_*.nc``. The output variable is ``ismip7ExcessMelt`` From 221526e1e512104bf3546b26a7a4a85a6a7ce716 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 24 Sep 2026 13:29:23 -0700 Subject: [PATCH 3/3] Fix path errors when processing OCX forcing --- .../atmosphere/build_mapping_file.py | 26 ++++++++++++++----- .../ocean_thermal/build_mapping_file.py | 12 ++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py index 73520b3aa7..f990d58a6a 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py @@ -85,12 +85,26 @@ def run(self): # Use any atmosphere file as the grid template for building the # mapping file. We'll use the first year's acabf (SMB) file. - prefix = params['prefix'] - resolution = params['atm_resolution'] - version = params['atm_version'] - model = (section.get("model") if params['atm_model'] is None - else params['atm_model']) - scenario = section.get("scenario") + ismip7_section = config["ismip7"] + base_path_ismip7 = ismip7_section.get("base_path_ismip7") + mali_mesh_name = ismip7_section.get("mali_mesh_name") + mali_mesh_file = ismip7_section.get("mali_mesh_file") + ice_sheet = ismip7_section.get("ice_sheet") + output_base_path = ismip7_section.get("output_base_path") + mapping_files_path = ismip7_section.get("mapping_files_path") + + atmosphere_section = config["ismip7_atmosphere"] + method_remap = atmosphere_section.get("method_remap") + + prefix = params["prefix"] + resolution = params["atm_resolution"] + version = params["atm_version"] + + model = (ismip7_section.get("model") + if params["atm_model"] is None + else params["atm_model"]) + + scenario = ismip7_section.get("scenario") input_path = os.path.join(base_path_ismip7, "acabf", version) # Use a simple pattern that will match the first available file diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py index cc09fed192..9682bd70a8 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py @@ -98,8 +98,8 @@ def run(self): # Use 'main' as the default choice for building the mapping file ocean_choice = 'main' input_path = os.path.join( - base_path_ismip7, "thermal_forcing", version, ocean_choice) - file_pattern = (f"thermal_forcing_{prefix}_{ocean_grid}_" + base_path_ismip7, "ocean", ocean_choice, version) + file_pattern = (f"tf_{prefix}_OCX_{ocean_grid}_{ocean_choice}_" f"{version}_*.nc") else: # Standard ESM scenario or GrIS OCX @@ -107,14 +107,14 @@ def run(self): # OCX (GrIS): use the ocean_model token ocean_model = params['ocean_model'] input_path = os.path.join( - base_path_ismip7, "thermal_forcing", version) - file_pattern = (f"thermal_forcing_{prefix}_{ocean_model}_" + base_path_ismip7, "ocean", "tf", version) + file_pattern = (f"tf_{prefix}_{ocean_model}_" f"{ocean_grid}_{version}_*.nc") else: # Standard ESM scenario input_path = os.path.join( - base_path_ismip7, "thermal_forcing", version) - file_pattern = (f"thermal_forcing_{prefix}_{model}_" + base_path_ismip7, "ocean", version) + file_pattern = (f"tf_{prefix}_{model}_" f"{scenario}_{ocean_grid}_{version}_*.nc") ismip7_grid_files = sorted(