From 4eeada9c3f12889909ab4ae9ca13d5a659d751de Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Tue, 29 Sep 2026 12:14:08 -0600 Subject: [PATCH 1/6] Skip ismip7_forcing steps when final output already exists When an ismip7_forcing processing step runs, it previously always regenerated and overwrote its final output. The only resume logic was at the temporary remapped_* level, which is deleted once a step completes. This meant re-running a test case (after a partial timeout, or to process a different scenario) re-did all expensive extrapolate + ncremap work for steps that already finished. This commit adds an early 'final output already exists -> info-log and skip' guard to every forcing step that produces a final output, making re-runs cheap and idempotent. Changes: - Add os.path.exists(dst) check at the start of each run() method, after source resolution but before any remap work - Hoist output_file/output_path/dst computation to before the guard so the tail of run() reuses those variables - Guard placement ensures each output is checked independently (e.g. ocean_thermal per-job outputs for AIS OCX ocean choices) Affected steps (10): - Atmosphere: process_smb, process_temperature, process_runoff, process_smb_gradient, process_temperature_gradient - Ocean thermal: process_thermal_forcing (_process_ocean_forcing and _run_climatology), build_3d_thermal_forcing - Fracture: process_excess_melt, process_lake_properties, process_shelf_collapse The existing per-year remapped_* skip logic is unchanged; these guards operate at the final canonical output level. Co-Authored-By: Claude Sonnet 4.5 --- .../atmosphere/process_runoff.py | 16 +++++---- .../ismip7_forcing/atmosphere/process_smb.py | 16 +++++---- .../atmosphere/process_smb_gradient.py | 16 +++++---- .../atmosphere/process_temperature.py | 16 +++++---- .../process_temperature_gradient.py | 16 +++++---- .../fracture/process_excess_melt.py | 13 ++++--- .../fracture/process_lake_properties.py | 13 ++++--- .../fracture/process_shelf_collapse.py | 13 ++++--- .../ocean_thermal/build_3d_thermal_forcing.py | 5 +++ .../ocean_thermal/process_thermal_forcing.py | 36 +++++++++++-------- 10 files changed, 104 insertions(+), 56 deletions(-) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py index 94db1faf9c..df8a664f5e 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py @@ -93,6 +93,16 @@ def run(self): logger.info(f"Found {len(input_files)} runoff files for years " f"{start_year}-{end_year}") + # Check if final output already exists; skip if so + output_file = (f"{mali_mesh_name}_runoff_{model}_{scenario}_" + f"{start_year}-{end_year}.nc") + output_path = os.path.join(output_base_path, forcing_group, + "atmosphere") + dst = os.path.join(output_path, output_file) + if os.path.exists(dst): + logger.info(f"Output already exists, skipping: {dst}") + return + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "atm", source.source_grid, method_remap) @@ -129,9 +139,6 @@ def run(self): # Combine remapped files and rename to MALI conventions logger.info("Combining remapped files and renaming variables...") - output_file = (f"{mali_mesh_name}_runoff_{model}_{scenario}_" - f"{start_year}-{end_year}.nc") - self._combine_and_rename(remapped_files, output_file) # Clean up remapped files @@ -141,12 +148,9 @@ def run(self): os.remove(f) # Place output in appropriate directory - output_path = os.path.join(output_base_path, forcing_group, - "atmosphere") if not os.path.exists(output_path): os.makedirs(output_path) - dst = os.path.join(output_path, output_file) shutil.copy(output_file, dst) logger.info(f"Done. Output: {dst}") diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py index e84ea3c0c0..d88bed6a8c 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py @@ -93,6 +93,16 @@ def run(self): logger.info(f"Found {len(input_files)} SMB files for years " f"{start_year}-{end_year}") + # Check if final output already exists; skip if so + output_file = (f"{mali_mesh_name}_SMB_{model}_{scenario}_" + f"{start_year}-{end_year}.nc") + output_path = os.path.join(output_base_path, forcing_group, + "atmosphere") + dst = os.path.join(output_path, output_file) + if os.path.exists(dst): + logger.info(f"Output already exists, skipping: {dst}") + return + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "atm", source.source_grid, method_remap) @@ -128,9 +138,6 @@ def run(self): # Combine remapped files and rename to MALI conventions logger.info("Combining remapped files and renaming variables...") - output_file = (f"{mali_mesh_name}_SMB_{model}_{scenario}_" - f"{start_year}-{end_year}.nc") - self._combine_and_rename(remapped_files, output_file) # Clean up remapped files @@ -140,12 +147,9 @@ def run(self): os.remove(f) # Place output in appropriate directory - output_path = os.path.join(output_base_path, forcing_group, - "atmosphere") if not os.path.exists(output_path): os.makedirs(output_path) - dst = os.path.join(output_path, output_file) shutil.copy(output_file, dst) logger.info(f"Done. Output: {dst}") diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py index 9f38bde8d7..be0277cef5 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py @@ -94,6 +94,16 @@ def run(self): logger.info(f"Found {len(input_files)} SMB gradient files for years " f"{start_year}-{end_year}") + # Check if final output already exists; skip if so + output_file = (f"{mali_mesh_name}_SMB_gradient_{model}_{scenario}_" + f"{start_year}-{end_year}.nc") + output_path = os.path.join(output_base_path, forcing_group, + "atmosphere") + dst = os.path.join(output_path, output_file) + if os.path.exists(dst): + logger.info(f"Output already exists, skipping: {dst}") + return + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "atm", source.source_grid, method_remap) @@ -130,9 +140,6 @@ def run(self): # Combine remapped files and rename to MALI conventions logger.info("Combining remapped files and renaming variables...") - output_file = (f"{mali_mesh_name}_SMB_gradient_{model}_{scenario}_" - f"{start_year}-{end_year}.nc") - self._combine_and_rename(remapped_files, output_file) # Clean up remapped files @@ -142,12 +149,9 @@ def run(self): os.remove(f) # Place output in appropriate directory - output_path = os.path.join(output_base_path, forcing_group, - "atmosphere") if not os.path.exists(output_path): os.makedirs(output_path) - dst = os.path.join(output_path, output_file) shutil.copy(output_file, dst) logger.info(f"Done. Output: {dst}") diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py index dfd145fe7c..9a7349ff40 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py @@ -93,6 +93,16 @@ def run(self): logger.info(f"Found {len(input_files)} temperature files for years " f"{start_year}-{end_year}") + # Check if final output already exists; skip if so + output_file = (f"{mali_mesh_name}_temperature_{model}_{scenario}_" + f"{start_year}-{end_year}.nc") + output_path = os.path.join(output_base_path, forcing_group, + "atmosphere") + dst = os.path.join(output_path, output_file) + if os.path.exists(dst): + logger.info(f"Output already exists, skipping: {dst}") + return + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "atm", source.source_grid, method_remap) @@ -128,9 +138,6 @@ def run(self): # Combine remapped files and rename to MALI conventions logger.info("Combining remapped files and renaming variables...") - output_file = (f"{mali_mesh_name}_temperature_{model}_{scenario}_" - f"{start_year}-{end_year}.nc") - self._combine_and_rename(remapped_files, output_file) # Clean up remapped files @@ -140,12 +147,9 @@ def run(self): os.remove(f) # Place output in appropriate directory - output_path = os.path.join(output_base_path, forcing_group, - "atmosphere") if not os.path.exists(output_path): os.makedirs(output_path) - dst = os.path.join(output_path, output_file) shutil.copy(output_file, dst) logger.info(f"Done. Output: {dst}") diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py index 47541a0798..5d0bbd55af 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py @@ -95,6 +95,16 @@ def run(self): logger.info(f"Found {len(input_files)} temperature gradient files " f"for years {start_year}-{end_year}") + # Check if final output already exists; skip if so + output_file = (f"{mali_mesh_name}_temperature_gradient_{model}_" + f"{scenario}_{start_year}-{end_year}.nc") + output_path = os.path.join(output_base_path, forcing_group, + "atmosphere") + dst = os.path.join(output_path, output_file) + if os.path.exists(dst): + logger.info(f"Output already exists, skipping: {dst}") + return + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "atm", source.source_grid, method_remap) @@ -130,9 +140,6 @@ def run(self): # Combine remapped files and rename to MALI conventions logger.info("Combining remapped files and renaming variables...") - output_file = (f"{mali_mesh_name}_temperature_gradient_{model}_" - f"{scenario}_{start_year}-{end_year}.nc") - self._combine_and_rename(remapped_files, output_file) # Clean up remapped files @@ -142,12 +149,9 @@ def run(self): os.remove(f) # Place output in appropriate directory - output_path = os.path.join(output_base_path, forcing_group, - "atmosphere") if not os.path.exists(output_path): os.makedirs(output_path) - dst = os.path.join(output_path, output_file) shutil.copy(output_file, dst) logger.info(f"Done. Output: {dst}") diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py index 9a68fcd753..ff61d2c7ff 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py @@ -104,6 +104,15 @@ def run(self): basename = os.path.basename(input_file) logger.info(f"Processing excess melt: {basename}") + # Check if final output already exists; skip if so + output_file = f"{mali_mesh_name}_{basename}" + output_path = os.path.join(output_base_path, f"{model}_{scenario}", + "excess_melt") + dst = os.path.join(output_path, output_file) + if os.path.exists(dst): + logger.info(f"Output already exists, skipping: {dst}") + return + # Build a source file with x/y coordinates and correct orientation gridded_file = f"gridded_{basename}" self._prepare_source_grid(input_file, input_path, gridded_file, @@ -131,7 +140,6 @@ def run(self): # Rename to MALI conventions logger.info("Renaming variables to MALI conventions...") - output_file = f"{mali_mesh_name}_{basename}" self._rename_to_mali_vars(remapped_file, output_file, start_year, end_year) @@ -141,12 +149,9 @@ def run(self): os.remove(f) # Place output in the appropriate directory - output_path = os.path.join(output_base_path, f"{model}_{scenario}", - "excess_melt") if not os.path.exists(output_path): os.makedirs(output_path) - dst = os.path.join(output_path, output_file) shutil.copy(output_file, dst) logger.info(f"Done. Output: {dst}") diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py index 839af87ea7..b21c1dcc28 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py @@ -107,6 +107,15 @@ def run(self): basename = os.path.basename(input_file) logger.info(f"Processing lake properties: {basename}") + # Check if final output already exists; skip if so + output_file = f"{mali_mesh_name}_{basename}" + output_path = os.path.join(output_base_path, f"{model}_{scenario}", + "lake_properties") + dst = os.path.join(output_path, output_file) + if os.path.exists(dst): + logger.info(f"Output already exists, skipping: {dst}") + return + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "fracture", source.source_grid, method_remap) @@ -130,7 +139,6 @@ def run(self): # Rename to MALI conventions logger.info("Renaming variables to MALI conventions...") - output_file = f"{mali_mesh_name}_{basename}" self._rename_to_mali_vars(remapped_file, output_file, start_year, end_year) @@ -140,12 +148,9 @@ def run(self): os.remove(f) # Place output in the appropriate directory - output_path = os.path.join(output_base_path, f"{model}_{scenario}", - "lake_properties") if not os.path.exists(output_path): os.makedirs(output_path) - dst = os.path.join(output_path, output_file) shutil.copy(output_file, dst) logger.info(f"Done. Output: {dst}") diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py index df5d1b6d7c..b41ebab44c 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py @@ -94,6 +94,15 @@ def run(self): basename = os.path.basename(input_file) logger.info(f"Processing ice shelf collapse mask: {basename}") + # Check if final output already exists; skip if so + output_file = f"{mali_mesh_name}_{basename}" + output_path = os.path.join(output_base_path, f"{model}_{scenario}", + "shelf_collapse") + dst = os.path.join(output_path, output_file) + if os.path.exists(dst): + logger.info(f"Output already exists, skipping: {dst}") + return + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "fracture", source.source_grid, method_remap) @@ -111,7 +120,6 @@ def run(self): # Combine time slice and rename to MALI conventions logger.info("Renaming variables to MALI conventions...") - output_file = f"{mali_mesh_name}_{basename}" self._rename_to_mali_vars(remapped_file, output_file, start_year, end_year) @@ -120,12 +128,9 @@ def run(self): os.remove(remapped_file) # Place output in the appropriate directory - output_path = os.path.join(output_base_path, f"{model}_{scenario}", - "shelf_collapse") if not os.path.exists(output_path): os.makedirs(output_path) - dst = os.path.join(output_path, output_file) shutil.copy(output_file, dst) logger.info(f"Done. Output: {dst}") diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py index ec2a2173a0..8fcdc34285 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py @@ -77,6 +77,11 @@ def run(self): f"{mali_mesh_name}_3dThermalForcing_{label}_" f"{start_year}-{end_year}.nc") + # Check if final output already exists; skip if so + if os.path.exists(output_file): + logger.info(f"Output already exists, skipping: {output_file}") + return + # DeltaT/gamma0/basin are calibrated once against OCX and held fixed # for every ESM (see Config.calibrate_delta_t), so they always live # under the OCX output directory regardless of which scenario this diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py index b897a2ae69..c67b7b76f3 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py @@ -255,6 +255,17 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, label = job.label all_files = job.files + # Check if final output already exists; skip if so + tf_label = "3dThermalForcing" if ocean_3d else "2dThermalForcing" + output_file = (f"{mali_mesh_name}_{tf_label}_{label}_" + f"{start_year}-{end_year}.nc") + output_path = os.path.join(output_base_path, forcing_group, + "ocean_thermal_forcing") + dst = os.path.join(output_path, output_file) + if os.path.exists(dst): + logger.info(f"Output already exists, skipping: {dst}") + return + # Filter to files that overlap with the requested year range. # AIS files are named with decade or multi-decade ranges (e.g., # 1850-1859, 1950-2025). GrIS files are named with single years. @@ -308,10 +319,6 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, # Combine remapped files and rename to MALI conventions logger.info("Combining remapped files and renaming variables...") - tf_label = "3dThermalForcing" if ocean_3d else "2dThermalForcing" - output_file = (f"{mali_mesh_name}_{tf_label}_{label}_" - f"{start_year}-{end_year}.nc") - if ocean_3d: self._combine_and_rename_3d(remapped_files, output_file, start_year, end_year) @@ -326,12 +333,9 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, os.remove(f) # Place output in appropriate directory - output_path = os.path.join(output_base_path, forcing_group, - "ocean_thermal_forcing") if not os.path.exists(output_path): os.makedirs(output_path) - dst = os.path.join(output_path, output_file) shutil.copy(output_file, dst) logger.info(f"Done. Output: {dst}") @@ -364,6 +368,17 @@ def _run_climatology(self): logger.info(f"Processing ocean TF climatology: " f"{os.path.basename(input_file)}") + # Check if final output already exists; skip if so + version = os.path.basename(os.path.dirname(input_file)) + output_file = (f"{mali_mesh_name}_thermal_forcing_climatology_" + f"{version}.nc") + output_path = os.path.join(output_base_path, + "ocean_thermal_forcing", "climatology") + dst = os.path.join(output_path, output_file) + if os.path.exists(dst): + logger.info(f"Output already exists, skipping: {dst}") + return + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "ocean", "climatology", method_remap) @@ -392,10 +407,6 @@ def _run_climatology(self): # Rename to MALI conventions logger.info("Renaming variables to MALI conventions...") - version = os.path.basename(os.path.dirname(input_file)) - output_file = (f"{mali_mesh_name}_thermal_forcing_climatology_" - f"{version}.nc") - self._rename_climatology_3d(remapped_file, output_file) # Clean up remapped file @@ -403,12 +414,9 @@ def _run_climatology(self): os.remove(remapped_file) # Place output in appropriate directory - output_path = os.path.join(output_base_path, "ocean_thermal_forcing", - "climatology") if not os.path.exists(output_path): os.makedirs(output_path) - dst = os.path.join(output_path, output_file) shutil.copy(output_file, dst) logger.info(f"Done. Output: {dst}") From 30c3cad35c2c26dc0313c82bb680c450072d8eae Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Tue, 29 Sep 2026 11:21:50 -0700 Subject: [PATCH 2/6] Fix bug in config parser when determining fracture forcing version --- compass/landice/ismip7/archive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compass/landice/ismip7/archive.py b/compass/landice/ismip7/archive.py index 1e4d773f85..87e951bb41 100644 --- a/compass/landice/ismip7/archive.py +++ b/compass/landice/ismip7/archive.py @@ -149,8 +149,8 @@ def resolve_fracture_source(config, file_pattern): scenario = config.get('ismip7', 'scenario') base_path = os.path.join( _ice_sheet_path(config), model, scenario, 'fracture') - requested_version = config.get( - 'ismip7_fracture', 'version', fallback='latest') + requested_version = config['ismip7_fracture'].get( + 'version', fallback='latest') version_path, version, files = resolve_version_directory( base_path, requested_version, file_pattern) From 90fc71a67ce898ebcec8872d378cbb3d22285414 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Tue, 29 Sep 2026 14:23:21 -0600 Subject: [PATCH 3/6] Fix 3-D thermal forcing completion check to use actual chunk filename The greenland_3d.run() function writes per-chunk output files named by their start year (e.g., GrIS_..._3dThermalForcing_OCX_2007.nc, ..._2017.nc), not a single file with the full year range. The previous guard checked for the non-existent full-range filename, so it never skipped even after a successful run. This commit fixes the guard to check for the first chunk file (with start_year only, not start_year-end_year), which greenland_3d.run actually produces. The check is conservative: it only verifies the first chunk exists, consistent with the per-year remapped file skip pattern used in the other ismip7_forcing steps. Addresses Copilot review comment on PR #1001. Co-Authored-By: Claude Sonnet 4.5 --- .../ocean_thermal/build_3d_thermal_forcing.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py index 8fcdc34285..6cbd360e82 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py @@ -77,9 +77,15 @@ def run(self): f"{mali_mesh_name}_3dThermalForcing_{label}_" f"{start_year}-{end_year}.nc") - # Check if final output already exists; skip if so - if os.path.exists(output_file): - logger.info(f"Output already exists, skipping: {output_file}") + # Check if output already exists; greenland_3d.run writes per-chunk + # files named by start year, not the full range, so check first chunk + first_chunk = os.path.join( + ocean_dir, + f"{mali_mesh_name}_3dThermalForcing_{label}_" + f"{start_year}.nc") + if os.path.exists(first_chunk): + logger.info(f"Output already exists (first chunk found), " + f"skipping: {first_chunk}") return # DeltaT/gamma0/basin are calibrated once against OCX and held fixed From 9994e92d6d151c3f79ee324b1edb3fe1432e8cc7 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Tue, 29 Sep 2026 14:46:23 -0600 Subject: [PATCH 4/6] Validate and reprocess incomplete intermediate files on resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an ismip7_forcing job times out mid-remap, it can leave truncated or incomplete remapped_* and extrap_* files. The existing per-file resume logic reused these based purely on os.path.exists(), silently feeding bad intermediates into the final concatenation, which then failed with 'ValueError: coordinate time not present in all datasets'. This commit adds netcdf_file_is_valid() to compass/landice/ismip7/remap.py, which checks that a file exists, opens cleanly, contains the expected variable and (optionally) time coordinate, and can actually read its last time record (catches header-complete but data-truncated files that open cleanly but fail later). Each step's per-file skip guard is now: 'file exists AND valid → skip; otherwise delete and regenerate.' Coverage: - State 1: file never created → os.path.exists catches it - State 2: truncated header (bad magic bytes) → raises on open - State 3: header OK but variable/time missing → presence check - State 4: header complete but data truncated → last-record .load() raises (this closes the subtle gap where lazy xarray reads only the header and the truncated file passes an open+presence check) Changes: - Add netcdf_file_is_valid() helper to remap.py - Update all 5 atmosphere steps (SMB, temperature, runoff, gradients) to validate both remapped_* (require_time=True) and extrap_* files - Update ocean_thermal/_process_ocean_forcing (tf, require_time=True) and _run_climatology (tf, require_time=False — static 3-D field) - Update fracture/process_shelf_collapse (mask, require_time=True) No change to excess_melt / lake_properties — those regenerate intermediates unconditionally (no exists-guard to reuse a stale file). Fixes the reported 'time not present' concat error and covers all four interrupted-write states at negligible cost (single-slice read). Co-Authored-By: Claude Sonnet 4.5 --- compass/landice/ismip7/remap.py | 54 +++++++++++++++++++ .../atmosphere/process_runoff.py | 22 ++++++-- .../ismip7_forcing/atmosphere/process_smb.py | 19 +++++-- .../atmosphere/process_smb_gradient.py | 19 +++++-- .../atmosphere/process_temperature.py | 18 +++++-- .../process_temperature_gradient.py | 19 +++++-- .../fracture/process_shelf_collapse.py | 8 ++- .../ocean_thermal/process_thermal_forcing.py | 33 ++++++++---- 8 files changed, 165 insertions(+), 27 deletions(-) diff --git a/compass/landice/ismip7/remap.py b/compass/landice/ismip7/remap.py index 720ef39e85..6f6eaa4459 100644 --- a/compass/landice/ismip7/remap.py +++ b/compass/landice/ismip7/remap.py @@ -10,6 +10,60 @@ from scipy.ndimage import distance_transform_edt +def netcdf_file_is_valid(path, varname=None, require_time=False, + logger=None): + """ + Return True if ``path`` exists, opens cleanly, contains ``varname`` and + (optionally) a ``time`` coordinate, and its last time record can + actually be read. Used to detect incomplete intermediate files left by + an interrupted (timed-out) run so they can be regenerated instead of + silently reused. The last-record read catches a file whose header is + complete but whose data section was truncated mid-write (which opens + cleanly but fails later at concatenation). + + Parameters + ---------- + path : str + Path to the NetCDF file to check + + varname : str, optional + Variable name to verify is present; if None, only checks file opens + + require_time : bool, optional + Whether to require a ``time`` coordinate/variable + + logger : logging.Logger, optional + Logger for status messages (logs incomplete-file info if provided) + + Returns + ------- + bool + True if the file is valid and complete; False otherwise + """ + if not os.path.exists(path): + return False + try: + with xr.open_dataset(path, decode_times=False) as ds: + if varname is not None and varname not in ds.variables: + return False + if require_time and "time" not in ds.variables: + return False + # Force-read the last record to detect a truncated data + # section (header OK, but bytes cut off mid-file). + if varname is not None: + var = ds[varname] + if "time" in var.dims: + var.isel(time=-1).load() + else: + var.load() + except Exception as exc: + if logger is not None: + logger.info(f" Ignoring incomplete/unreadable file " + f"{os.path.basename(path)}: {exc}") + return False + return True + + def extrapolate_source(input_file, output_file, varnames, logger): """ Extrapolate fill/missing values on the source polar stereographic grid diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py index df8a664f5e..e909526a5a 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py @@ -9,7 +9,10 @@ mapping_file_name, resolve_atmosphere_source, ) -from compass.landice.ismip7.remap import extrapolate_source +from compass.landice.ismip7.remap import ( + extrapolate_source, + netcdf_file_is_valid, +) from compass.step import Step @@ -114,16 +117,25 @@ def run(self): remapped_file = f"remapped_{basename}" remapped_files.append(remapped_file) - if os.path.exists(remapped_file): + if netcdf_file_is_valid(remapped_file, "mrro", + require_time=True, logger=logger): logger.info(f" Remapped file exists, skipping: {basename}") continue + # Incomplete remapped file from an interrupted run: drop and + # redo + if os.path.exists(remapped_file): + logger.info(f" Reprocessing incomplete remapped file: " + f"{basename}") + os.remove(remapped_file) # Extrapolate fill values on source grid before remapping # so they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" - if not os.path.exists(extrap_file): - extrapolate_source(input_file, extrap_file, "mrro", - logger) + if not netcdf_file_is_valid(extrap_file, "mrro", + logger=logger): + if os.path.exists(extrap_file): + os.remove(extrap_file) + extrapolate_source(input_file, extrap_file, "mrro", logger) logger.info(f" Remapping: {basename}") args = ["ncremap", diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py index d88bed6a8c..40b111553b 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py @@ -9,7 +9,10 @@ mapping_file_name, resolve_atmosphere_source, ) -from compass.landice.ismip7.remap import extrapolate_source +from compass.landice.ismip7.remap import ( + extrapolate_source, + netcdf_file_is_valid, +) from compass.step import Step @@ -114,14 +117,24 @@ def run(self): remapped_file = f"remapped_{basename}" remapped_files.append(remapped_file) - if os.path.exists(remapped_file): + if netcdf_file_is_valid(remapped_file, "acabf", + require_time=True, logger=logger): logger.info(f" Remapped file exists, skipping: {basename}") continue + # Incomplete remapped file from an interrupted run: drop and + # redo + if os.path.exists(remapped_file): + logger.info(f" Reprocessing incomplete remapped file: " + f"{basename}") + os.remove(remapped_file) # Extrapolate fill values on source grid before remapping # so they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" - if not os.path.exists(extrap_file): + if not netcdf_file_is_valid(extrap_file, "acabf", + logger=logger): + if os.path.exists(extrap_file): + os.remove(extrap_file) extrapolate_source(input_file, extrap_file, "acabf", logger) logger.info(f" Remapping: {basename}") diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py index be0277cef5..e60ede37da 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py @@ -9,7 +9,10 @@ mapping_file_name, resolve_atmosphere_source, ) -from compass.landice.ismip7.remap import extrapolate_source +from compass.landice.ismip7.remap import ( + extrapolate_source, + netcdf_file_is_valid, +) from compass.step import Step @@ -115,14 +118,24 @@ def run(self): remapped_file = f"remapped_{basename}" remapped_files.append(remapped_file) - if os.path.exists(remapped_file): + if netcdf_file_is_valid(remapped_file, "dacabfdz", + require_time=True, logger=logger): logger.info(f" Remapped file exists, skipping: {basename}") continue + # Incomplete remapped file from an interrupted run: drop and + # redo + if os.path.exists(remapped_file): + logger.info(f" Reprocessing incomplete remapped file: " + f"{basename}") + os.remove(remapped_file) # Extrapolate fill values on source grid before remapping # so they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" - if not os.path.exists(extrap_file): + if not netcdf_file_is_valid(extrap_file, "dacabfdz", + logger=logger): + if os.path.exists(extrap_file): + os.remove(extrap_file) extrapolate_source(input_file, extrap_file, "dacabfdz", logger) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py index 9a7349ff40..0001c51ee6 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py @@ -9,7 +9,10 @@ mapping_file_name, resolve_atmosphere_source, ) -from compass.landice.ismip7.remap import extrapolate_source +from compass.landice.ismip7.remap import ( + extrapolate_source, + netcdf_file_is_valid, +) from compass.step import Step @@ -114,14 +117,23 @@ def run(self): remapped_file = f"remapped_{basename}" remapped_files.append(remapped_file) - if os.path.exists(remapped_file): + if netcdf_file_is_valid(remapped_file, "ts", require_time=True, + logger=logger): logger.info(f" Remapped file exists, skipping: {basename}") continue + # Incomplete remapped file from an interrupted run: drop and + # redo + if os.path.exists(remapped_file): + logger.info(f" Reprocessing incomplete remapped file: " + f"{basename}") + os.remove(remapped_file) # Extrapolate fill values on source grid before remapping # so they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" - if not os.path.exists(extrap_file): + if not netcdf_file_is_valid(extrap_file, "ts", logger=logger): + if os.path.exists(extrap_file): + os.remove(extrap_file) extrapolate_source(input_file, extrap_file, "ts", logger) logger.info(f" Remapping: {basename}") diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py index 5d0bbd55af..c8910ff15e 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py @@ -9,7 +9,10 @@ mapping_file_name, resolve_atmosphere_source, ) -from compass.landice.ismip7.remap import extrapolate_source +from compass.landice.ismip7.remap import ( + extrapolate_source, + netcdf_file_is_valid, +) from compass.step import Step @@ -116,14 +119,24 @@ def run(self): remapped_file = f"remapped_{basename}" remapped_files.append(remapped_file) - if os.path.exists(remapped_file): + if netcdf_file_is_valid(remapped_file, "dtsdz", + require_time=True, logger=logger): logger.info(f" Remapped file exists, skipping: {basename}") continue + # Incomplete remapped file from an interrupted run: drop and + # redo + if os.path.exists(remapped_file): + logger.info(f" Reprocessing incomplete remapped file: " + f"{basename}") + os.remove(remapped_file) # Extrapolate fill values on source grid before remapping # so they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" - if not os.path.exists(extrap_file): + if not netcdf_file_is_valid(extrap_file, "dtsdz", + logger=logger): + if os.path.exists(extrap_file): + os.remove(extrap_file) extrapolate_source(input_file, extrap_file, "dtsdz", logger) logger.info(f" Remapping: {basename}") diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py index b41ebab44c..5055e0fe27 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py @@ -9,6 +9,7 @@ ) from compass.landice.ismip7.remap import ( add_xtime_and_write, + netcdf_file_is_valid, open_rename_and_trim, ) from compass.step import Step @@ -109,7 +110,12 @@ def run(self): # Remap the collapse mask onto the MALI mesh remapped_file = f"remapped_{basename}" - if not os.path.exists(remapped_file): + if not netcdf_file_is_valid(remapped_file, "mask", + require_time=True, logger=logger): + if os.path.exists(remapped_file): + logger.info(f"Reprocessing incomplete remapped file: " + f"{basename}") + os.remove(remapped_file) logger.info(f"Remapping: {basename}") args = ["ncremap", "-i", input_file, diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py index c67b7b76f3..dcd3194974 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py @@ -11,7 +11,10 @@ resolve_version_directory, ) from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.remap import extrapolate_source +from compass.landice.ismip7.remap import ( + extrapolate_source, + netcdf_file_is_valid, +) from compass.step import Step @@ -294,16 +297,24 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, remapped_file = f"remapped_{basename}" remapped_files.append(remapped_file) - if os.path.exists(remapped_file): + if netcdf_file_is_valid(remapped_file, "tf", require_time=True, + logger=logger): logger.info(f" Remapped file exists, skipping: {basename}") continue + # Incomplete remapped file from an interrupted run: drop and + # redo + if os.path.exists(remapped_file): + logger.info(f" Reprocessing incomplete remapped file: " + f"{basename}") + os.remove(remapped_file) # Extrapolate fill values on source grid before remapping # so they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" - if not os.path.exists(extrap_file): - extrapolate_source(input_file, extrap_file, "tf", - logger) + if not netcdf_file_is_valid(extrap_file, "tf", logger=logger): + if os.path.exists(extrap_file): + os.remove(extrap_file) + extrapolate_source(input_file, extrap_file, "tf", logger) logger.info(f" Remapping: {basename}") args = ["ncremap", @@ -387,11 +398,15 @@ def _run_climatology(self): basename = os.path.basename(input_file) remapped_file = f"remapped_{basename}" - if not os.path.exists(remapped_file): + if not netcdf_file_is_valid(remapped_file, "tf", + require_time=False, logger=logger): + if os.path.exists(remapped_file): + os.remove(remapped_file) extrap_file = f"extrap_{basename}" - if not os.path.exists(extrap_file): - extrapolate_source(input_file, extrap_file, "tf", - logger) + if not netcdf_file_is_valid(extrap_file, "tf", logger=logger): + if os.path.exists(extrap_file): + os.remove(extrap_file) + extrapolate_source(input_file, extrap_file, "tf", logger) logger.info(f" Remapping: {basename}") args = ["ncremap", From aa20937803d87cdc470c9ff3678ff4b8bb199878 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Tue, 29 Sep 2026 15:18:54 -0600 Subject: [PATCH 5/6] Embed source version in output filenames; remove conflicting 3-D guard; document validation helper Copilot findings #1-4 were valid: 1. Source version not in filename (HIGH): The forcing archive versions each source (e.g. v1, v2, v2.1) as a directory, and ForcingSource.version carries it, but output filenames omitted it. Upgrading from v1 to v2 in the archive caused the 'skip if exists' guard to reuse stale v1 output. 2. 3-D guard mishandles partial runs (HIGH): The first_chunk guard skipped the whole step when only the first chunk existed, leaving later chunks unwritten after a timeout. greenland_3d already has its own PER-CHUNK skip logic that correctly resumes partial runs. 3. Overwrite config ignored (MEDIUM): The first_chunk guard ran before the JSON was loaded, defeating a user's 'overwrite=true' request. greenland_3d's own logic honors cfg.overwrite. 4. netcdf_file_is_valid missing from API (LOW): Function had a complete docstring but was not registered in the Sphinx autosummary. Changes: - Embed source.version (or job.version) in all output filenames, placed BEFORE the year range: {mesh}_SMB_{model}_{scenario}_{version}_{start}- {end}.nc. Placement before the year range preserves trailing patterns that downstream regexes anchor to (_\d{4}-\d{4}$, _(\d{4})\.nc$), so no changes needed in greenland_3d.py or ismip7_run/set_up_experiment.py. - Remove the step-level first_chunk guard from build_3d_thermal_forcing.py (the redundant/conflicting guard added in commit 90fc71a). Delegate to greenland_3d's own per-chunk skip/overwrite logic, which is finer-grained (handles partial runs correctly) and overwrite-aware (honors the JSON cfg.overwrite flag). - Add remap.netcdf_file_is_valid to docs/developers_guide/landice/api.rst (before remap.extrapolate_source, following source order). Affected filenames (10 steps): - Atmosphere: {version}_{start}-{end}.nc for SMB, temperature, runoff, gradients - Ocean: {version}_{start}-{end}.nc for 2D/3D thermal forcing; climatology already embedded version (unchanged) - Fracture: {stem}_{version}{ext} (no year range) Downstream globs (*SMB*.nc, *2dThermalForcing_*.nc) still match; no regex changes needed. Now a version bump produces a new filename -> guard doesn't fire -> output regenerates (fixes staleness hazard). The GrIS 3-D path now honors overwrite and resumes partial runs correctly. Co-Authored-By: Claude Sonnet 4.5 --- .../ismip7_forcing/atmosphere/process_runoff.py | 2 +- .../ismip7_forcing/atmosphere/process_smb.py | 2 +- .../atmosphere/process_smb_gradient.py | 2 +- .../atmosphere/process_temperature.py | 2 +- .../atmosphere/process_temperature_gradient.py | 3 ++- .../fracture/process_excess_melt.py | 3 ++- .../fracture/process_lake_properties.py | 3 ++- .../fracture/process_shelf_collapse.py | 3 ++- .../ocean_thermal/build_3d_thermal_forcing.py | 15 ++------------- .../ocean_thermal/process_thermal_forcing.py | 2 +- docs/developers_guide/landice/api.rst | 1 + 11 files changed, 16 insertions(+), 22 deletions(-) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py index e909526a5a..f1976b03bb 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py @@ -98,7 +98,7 @@ def run(self): # Check if final output already exists; skip if so output_file = (f"{mali_mesh_name}_runoff_{model}_{scenario}_" - f"{start_year}-{end_year}.nc") + f"{source.version}_{start_year}-{end_year}.nc") output_path = os.path.join(output_base_path, forcing_group, "atmosphere") dst = os.path.join(output_path, output_file) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py index 40b111553b..d169dda839 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py @@ -98,7 +98,7 @@ def run(self): # Check if final output already exists; skip if so output_file = (f"{mali_mesh_name}_SMB_{model}_{scenario}_" - f"{start_year}-{end_year}.nc") + f"{source.version}_{start_year}-{end_year}.nc") output_path = os.path.join(output_base_path, forcing_group, "atmosphere") dst = os.path.join(output_path, output_file) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py index e60ede37da..20473acf00 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py @@ -99,7 +99,7 @@ def run(self): # Check if final output already exists; skip if so output_file = (f"{mali_mesh_name}_SMB_gradient_{model}_{scenario}_" - f"{start_year}-{end_year}.nc") + f"{source.version}_{start_year}-{end_year}.nc") output_path = os.path.join(output_base_path, forcing_group, "atmosphere") dst = os.path.join(output_path, output_file) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py index 0001c51ee6..5c9cbf4a09 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py @@ -98,7 +98,7 @@ def run(self): # Check if final output already exists; skip if so output_file = (f"{mali_mesh_name}_temperature_{model}_{scenario}_" - f"{start_year}-{end_year}.nc") + f"{source.version}_{start_year}-{end_year}.nc") output_path = os.path.join(output_base_path, forcing_group, "atmosphere") dst = os.path.join(output_path, output_file) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py index c8910ff15e..5ebb001c9a 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py @@ -100,7 +100,8 @@ def run(self): # Check if final output already exists; skip if so output_file = (f"{mali_mesh_name}_temperature_gradient_{model}_" - f"{scenario}_{start_year}-{end_year}.nc") + f"{scenario}_{source.version}_" + f"{start_year}-{end_year}.nc") output_path = os.path.join(output_base_path, forcing_group, "atmosphere") dst = os.path.join(output_path, output_file) diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py index ff61d2c7ff..77916101d6 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py @@ -105,7 +105,8 @@ def run(self): logger.info(f"Processing excess melt: {basename}") # Check if final output already exists; skip if so - output_file = f"{mali_mesh_name}_{basename}" + stem, ext = os.path.splitext(basename) + output_file = f"{mali_mesh_name}_{stem}_{source.version}{ext}" output_path = os.path.join(output_base_path, f"{model}_{scenario}", "excess_melt") dst = os.path.join(output_path, output_file) diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py index b21c1dcc28..a22a4738b4 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py @@ -108,7 +108,8 @@ def run(self): logger.info(f"Processing lake properties: {basename}") # Check if final output already exists; skip if so - output_file = f"{mali_mesh_name}_{basename}" + stem, ext = os.path.splitext(basename) + output_file = f"{mali_mesh_name}_{stem}_{source.version}{ext}" output_path = os.path.join(output_base_path, f"{model}_{scenario}", "lake_properties") dst = os.path.join(output_path, output_file) diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py index 5055e0fe27..3d0f9e2625 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py @@ -96,7 +96,8 @@ def run(self): logger.info(f"Processing ice shelf collapse mask: {basename}") # Check if final output already exists; skip if so - output_file = f"{mali_mesh_name}_{basename}" + stem, ext = os.path.splitext(basename) + output_file = f"{mali_mesh_name}_{stem}_{source.version}{ext}" output_path = os.path.join(output_base_path, f"{model}_{scenario}", "shelf_collapse") dst = os.path.join(output_path, output_file) diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py index 6cbd360e82..fa7bebbb82 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_3d_thermal_forcing.py @@ -71,22 +71,11 @@ def run(self): forcing_2d = os.path.join( ocean_dir, f"{mali_mesh_name}_2dThermalForcing_{label}_" - f"{start_year}-{end_year}.nc") + f"{source.version}_{start_year}-{end_year}.nc") output_file = os.path.join( ocean_dir, f"{mali_mesh_name}_3dThermalForcing_{label}_" - f"{start_year}-{end_year}.nc") - - # Check if output already exists; greenland_3d.run writes per-chunk - # files named by start year, not the full range, so check first chunk - first_chunk = os.path.join( - ocean_dir, - f"{mali_mesh_name}_3dThermalForcing_{label}_" - f"{start_year}.nc") - if os.path.exists(first_chunk): - logger.info(f"Output already exists (first chunk found), " - f"skipping: {first_chunk}") - return + f"{source.version}_{start_year}-{end_year}.nc") # DeltaT/gamma0/basin are calibrated once against OCX and held fixed # for every ESM (see Config.calibrate_delta_t), so they always live diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py index dcd3194974..4f07d89a3e 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py @@ -261,7 +261,7 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, # Check if final output already exists; skip if so tf_label = "3dThermalForcing" if ocean_3d else "2dThermalForcing" output_file = (f"{mali_mesh_name}_{tf_label}_{label}_" - f"{start_year}-{end_year}.nc") + f"{job.version}_{start_year}-{end_year}.nc") output_path = os.path.join(output_base_path, forcing_group, "ocean_thermal_forcing") dst = os.path.join(output_path, output_file) diff --git a/docs/developers_guide/landice/api.rst b/docs/developers_guide/landice/api.rst index 9e362e97e7..71329383e9 100644 --- a/docs/developers_guide/landice/api.rst +++ b/docs/developers_guide/landice/api.rst @@ -34,6 +34,7 @@ ISMIP7 framework archive.resolve_version_directory archive.mapping_file_name mapping.build_mapping_file + remap.netcdf_file_is_valid remap.extrapolate_source remap.open_rename_and_trim remap.add_xtime_and_write From fd5ee599a15f8806de7b4e21bb4dedfd5a304745 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 1 Oct 2026 11:37:34 -0600 Subject: [PATCH 6/6] Fix 3-D resume and decode_times crash from PR review Set overwrite=false in greenland_3d_tf_config.json so completed 3-D thermal forcing chunks are skipped on rerun instead of rebuilt. Sweep stale *.partial files at the start of greenland_3d.run() and drop the now-redundant FileExistsError guards, so an interrupted write no longer blocks the next run. Remove the unsupported decode_times=False argument from the extrapolate_source calls in process_excess_melt and process_lake_properties, which raised TypeError on fracture Paths A/B. Co-Authored-By: Claude Sonnet 4.5 --- .../fracture/process_excess_melt.py | 3 +-- .../fracture/process_lake_properties.py | 3 +-- .../greenland_3d_tf_config.json | 2 +- .../ocean_thermal/greenland_3d.py | 21 ++++++++++--------- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py index 77916101d6..882bf513be 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py @@ -126,8 +126,7 @@ def run(self): # Extrapolate fill values on the source grid before remapping so # they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" - extrapolate_source(gridded_file, extrap_file, "excess_melt", - logger, decode_times=False) + extrapolate_source(gridded_file, extrap_file, "excess_melt", logger) # Remap the excess melt onto the MALI mesh remapped_file = f"remapped_{basename}" diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py index a22a4738b4..abb6301731 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py @@ -125,8 +125,7 @@ def run(self): # they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" extrapolate_source(input_file, extrap_file, - list(self._variables.keys()), logger, - decode_times=False) + list(self._variables.keys()), logger) # Remap both lake property variables onto the MALI mesh remapped_file = f"remapped_{basename}" diff --git a/compass/landice/tests/ismip7_forcing/greenland_3d_tf_config.json b/compass/landice/tests/ismip7_forcing/greenland_3d_tf_config.json index b1c90072ec..5dfd730ba7 100644 --- a/compass/landice/tests/ismip7_forcing/greenland_3d_tf_config.json +++ b/compass/landice/tests/ismip7_forcing/greenland_3d_tf_config.json @@ -46,5 +46,5 @@ "b_degC": 0.0901, "c_degC_per_m": 0.000753 }, - "overwrite": true + "overwrite": false } diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/greenland_3d.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/greenland_3d.py index 87edb2b623..4150e8dcaa 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/greenland_3d.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/greenland_3d.py @@ -1500,11 +1500,6 @@ def _write_melt_params( ) return temporary = output.with_name(output.name + ".partial") - if temporary.exists(): - raise FileExistsError( - f"Partial melt-parameters output already exists: {temporary}. " - "Remove or rename it after inspecting it." - ) delta_t_by_cell = regional_delta_t[basin_ids - 1] target = xr.Dataset( data_vars={ @@ -1687,11 +1682,6 @@ def _write_forcing_chunk( ) return temporary = output_path.with_name(output_path.name + ".partial") - if temporary.exists(): - raise FileExistsError( - f"Partial output already exists: {temporary}. Remove or rename " - "it after inspecting it." - ) hdf5_temporary = output_path.with_name(output_path.name + ".h5.partial") target = xr.Dataset( data_vars={ @@ -2400,6 +2390,17 @@ def _warn_if_seafloor_below_max_depth( def run(cfg: Config, logger, prepare_only: bool = False) -> None: + # Remove any leftover .partial files from a previous interrupted run. + # These are incomplete temps; the real outputs are named differently. + seen = set() + for d in (cfg.output_file.parent, cfg.melt_params_file.parent): + if d in seen: + continue + seen.add(d) + for stale in sorted(d.glob("*.partial")): + logger.info(f"Removing stale partial file: {stale}") + stale.unlink() + mesh, basin_ids = _load_mesh_and_basins(cfg) profiles = _build_regional_profiles(cfg, mesh, basin_ids, logger) _warn_if_seafloor_below_max_depth(cfg, mesh, basin_ids, profiles, logger)