Update for Euler and Cleanup

cases/icon-test-euler/config.yaml

@@ -1,55 +1,51 @@
-# Configuration file for the 'icon-async-test' case with ICON
+# Global ICON simulation initialized from ERA5 (IC only, no LBC).
 
-workflow: icon
+workflow: icon-global-era5
 constraint: EPYC_7H12
 run_on: cpu
 compute_queue: normal.4h
 ntasks_per_node: 12
 restart_step: PT6H
 startdate: 2018-01-01T00:00:00Z
-enddate: 2018-01-01T12:00:00Z
-
-eccodes_dir: ./input/eccodes_definitions
-iconremap_bin: ./ext/icontools/icontools/iconremap
-iconsub_bin: ./ext/icontools/icontools/iconsub
-latbc_filename: ifs_201801<d><h>_lbc.nc
-inidata_filename: ifs_init_2018010100.nc
-output_filename: NWP_LAM
+enddate:   2018-01-01T12:00:00Z
+era5_partab: ./cases/icon-euler-test/partab_era5_to_icon.txt
+era5_ic_runjob_filename: era5_ic_runjob.cfg
+inidata_prefix: "era5_init_R02B05_"
+inidata_nameformat: "%Y%m%d%H"
+inidata_filename_suffix: ".nc"
+output_filename: icon-test-euler
 filename_format: <output_filename>_DOM<physdom>_<ddhhmmss>
 lateral_boundary_grid_order: lateral_boundary
 
 walltime:
-    prepare_icon: '00:10:00'
-    icontools: '00:30:00'
-    icon: '00:30:00'
+  prepare_icon: "00:10:00"
+  era5_ic:      "00:40:00"
+  icon:         "00:30:00"
 
-meteo:
-    dir: ./input/meteo
-    prefix: ifs_
-    nameformat: '%Y%m%d%H'
-    suffix: .grb
-    inc: 3
+era5:
+  dir: ./input/era5
+  ml_filename:   "era5_ml_{ymd}.grib"  
+  sfc_filename:  "era5_surf_{ymd}.grib"
+  global_nudging: False
 
-icontools_runjobs:
-    - icontools_remap_ic_runjob.cfg
-    - icontools_remap_00_lbc_runjob.cfg
-    - icontools_remap_lbc_rest_runjob.cfg
+meteo:
+  dir: ./input/meteo
+  prefix: ifs_
+  nameformat: "%Y%m%d%H"
+  suffix: .grb
+  inc: 3
 
 input_files:
-    radiation_grid_filename: ./input/icon/grid/VERIFY_DOM_DOM01.parent.nc
-    dynamics_grid_filename: ./input/icon/grid/VERIFY_DOM_DOM01.nc
-    map_file_latbc: ./input/icon/grid/map_file.latbc
-    lateral_boundary_grid: ./input/icon/grid/lateral_boundary.grid.nc
-    extpar_filename: ./input/icon/grid/external_parameter_icon_VERIFY_DOM_DOM01_tiles.nc
-    cldopt_filename: ./input/icon/rad/rrtm_cldopt.nc
-    lrtm_filename: ./input/icon/rad/rrtmg_lw.nc
-    map_file_ana: ./input/icon/mapping/map_file.ana
+  dynamics_grid_filename:   ./input/icon/grid/iconR2B05-DOM01.nc
+  extpar_filename:          ./input/icon/grid/extpar_iconR2B05-DOM01.nc
+  lrtm_filename:            ./input/icon/rad/rrtmg_lw.nc
+  cldopt_filename:          ./input/icon/rad/rrtm_cldopt.nc
+  map_file_ana:             ./input/icon/mapping/map_file.ana
 
 icon:
-    binary_file: ./ext/icon/bin/icon
-    runjob_filename: icon_runjob.cfg
-    np_tot: 96
-    np_io: 1
-    np_restart: 1
-    np_prefetch: 1
-
+  binary_file: /cluster/work/climate/lroither/icon_c2sm/icon/bin/icon
+  runjob_filename: icon_runjob.cfg
+  np_tot: 96
+  np_io: 1
+  np_restart: 1
+  np_prefetch: 0 

cases/icon-test-euler/era5_ic_runjob.cfg

@@ -0,0 +1,192 @@
+#!/usr/bin/env bash
+#SBATCH --job-name=era5_ic_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}
+#SBATCH --time={cfg.walltime_era5_ic}
+#SBATCH --partition={cfg.compute_queue}
+#SBATCH --constraint={cfg.constraint}
+#SBATCH --ntasks=1
+#SBATCH --output={cfg.logfile}
+#SBATCH --open-mode=append
+#SBATCH --chdir={cfg.icon_work}
+
+# ---------------------------------------------------------------------
+# ERA5 -> ICON initial conditions (GLOBAL, IC only)
+#
+# This script is submitted by jobs/era5_ic.py.
+# It MUST write the final IC file exactly to:
+#   {inidata_filename}
+#
+# Design:
+# - Use case config for input locations + partab
+# - Use staged ICON files from prepare_icon (cfg.input_files_scratch_*)
+# - Keep all intermediate files inside cfg.icon_work
+# ---------------------------------------------------------------------
+#
+set -euo pipefail
+set -x
+ulimit -s unlimited
+
+# Optional: load modules needed for cdo/nco on Euler.
+# Adjust these to your environment/stack.
+module load stack/2024-06 || true
+module load cdo/2.2.2 nco/5.1.6 netcdf-c/4.9.2 || true
+
+# ---------------------------------------------------------------------
+# Inputs (from cfg)
+# ---------------------------------------------------------------------
+ERA5_DIR="{cfg.era5_dir}"
+ERA5_ML="{cfg.era5_ml_filename}"
+ERA5_SFC="{cfg.era5_sfc_filename}"
+# Replace {ymd} placeholder using values prepared in era5_ic.py
+ERA5_ML="${{ERA5_ML/\{ymd\}/{cfg.era5_ymd}}}"
+ERA5_SFC="${{ERA5_SFC/\{ymd\}/{cfg.era5_ymd}}}"
+PARTAB="{cfg.era5_partab}"
+GRID_TRI="{cfg.input_files_scratch_dynamics_grid_filename}"
+EXTPAR="{cfg.input_files_scratch_extpar_filename}"
+OUTFILE="{inidata_filename}"
+
+# ---------------------------------------------------------------------
+# Working directory cleanup
+# ---------------------------------------------------------------------
+work="{cfg.icon_work}/era5_ic_work"
+mkdir -p "$work"
+cd "$work"
+
+# ---------------------------------------------------------------------
+# 1) GRIB preprocessing: pick first timestep and convert to NetCDF
+# ---------------------------------------------------------------------
+#
+# Note: your colleague used splitsel,1; keep it to ensure we only have t0.
+cdo splitsel,1 "${{ERA5_DIR}}/${{ERA5_ML}}" era5_ml_
+cdo splitsel,1 "${{ERA5_DIR}}/${{ERA5_SFC}}" era5_sfc_
+
+# Find the produced split files (avoid hardcoding the exact suffix)
+ml_grb="$(ls -1 era5_ml_* | head -n 1)"
+sfc_grb="$(ls -1 era5_sfc_* | head -n 1)"
+cdo -t ecmwf -f nc copy "$ml_grb" era5_ml.nc
+cdo -t ecmwf -f nc copy "$sfc_grb" era5_sfc.nc
+cdo merge era5_ml.nc era5_sfc.nc era5_original.nc
+
+# Rename variables / metadata to ICON-consistent naming via partab
+# (Your partab is the "contract" between ERA5 and ICON naming expectations.)
+cdo setpartabn,"$PARTAB",convert era5_original.nc tmp.nc
+
+# Order variables (helps reproducibility and easier diffs)
+ncks tmp.nc data_in.nc
+rm -f tmp.nc era5_ml.nc era5_sfc.nc era5_original.nc "$ml_grb" "$sfc_grb"
+
+# ---------------------------------------------------------------------
+# 2) Build the ICON triangular grid description for CDO remapping
+# ---------------------------------------------------------------------
+cdo -s selgrid,1 "$GRID_TRI" triangular-grid.nc
+
+# ---------------------------------------------------------------------
+# 3) Land/sea-aware remapping for sensitive surface variables
+# ---------------------------------------------------------------------
+#
+# Input land-sea mask from renamed ERA5 data
+cdo selname,LSM data_in.nc LSM_in.nc
+ncrename -h -v LSM,FR_LAND LSM_in.nc
+
+# Output land fraction from EXTPAR
+cdo selname,FR_LAND "$EXTPAR" LSM_out_tmp.nc
+
+# Add time dimension and align time coordinate
+ncecat -O -u time LSM_out_tmp.nc LSM_out_tmp.nc
+ncks -h -A -v time LSM_in.nc LSM_out_tmp.nc
+
+# Build land/ocean masks (input and output)
+cdo -L setctomiss,0. -ltc,0.5  LSM_in.nc oceanmask_in.nc
+cdo -L setctomiss,0. -gec,0.5 LSM_in.nc landmask_in.nc
+cdo -L setctomiss,0. -ltc,0.5 LSM_out_tmp.nc oceanmask_out.nc
+cdo -L setctomiss,0. -gec,0.5 LSM_out_tmp.nc landmask_out.nc
+cdo setrtoc2,0.5,1.0,1,0 LSM_out_tmp.nc LSM_out.nc
+rm -f LSM_in.nc LSM_out_tmp.nc
+
+# Variables defined only on sea
+ncks -h -v SST,CI data_in.nc datasea_in.nc
+
+# Variables defined on both but should not mix land/sea
+ncks -h -v SKT,STL1,STL2,STL3,STL4,ALB_SNOW,W_SNOW,T_SNOW data_in.nc dataland_in.nc
+
+# Ocean part: mask -> fill -> remap -> unmask
+cdo div dataland_in.nc oceanmask_in.nc tmp1_land.nc
+cdo div datasea_in.nc oceanmask_in.nc tmp1_sea.nc
+cdo setmisstodis tmp1_land.nc tmp2_land.nc
+cdo setmisstodis tmp1_sea.nc tmp2_sea.nc
+cdo remapdis,triangular-grid.nc tmp2_land.nc tmp3_land.nc
+cdo remapdis,triangular-grid.nc tmp2_sea.nc tmp3_sea.nc
+cdo div tmp3_land.nc oceanmask_out.nc dataland_ocean_out.nc
+cdo div tmp3_sea.nc oceanmask_out.nc datasea_ocean_out.nc
+rm -f tmp*_land.nc tmp*_sea.nc oceanmask_in.nc oceanmask_out.nc
+
+# Land part: mask -> fill -> remap -> unmask
+cdo div dataland_in.nc landmask_in.nc tmp1.nc
+cdo setmisstodis tmp1.nc tmp2.nc
+cdo remapdis,triangular-grid.nc tmp2.nc tmp3.nc
+cdo div tmp3.nc landmask_out.nc dataland_land_out.nc
+rm -f tmp*.nc landmask_in.nc landmask_out.nc dataland_in.nc datasea_in.nc
+
+# Merge land+ocean contributions
+cdo ifthenelse LSM_out.nc dataland_land_out.nc dataland_ocean_out.nc dataland_out.nc
+rm -f dataland_ocean_out.nc dataland_land_out.nc
+
+# Remap all remaining variables together
+ncks -h -x -v SKT,STL1,STL2,STL3,STL4,SMIL1,SMIL2,SMIL3,SMIL4,ALB_SNOW,W_SNOW,T_SNOW,SST,CI,LSM data_in.nc datarest_in.nc
+cdo -s remapdis,triangular-grid.nc datarest_in.nc era5_final.nc
+rm -f datarest_in.nc
+
+# Fill missing SST/CI after remap (sea-only fields)
+cdo setmisstodis -selname,SST,CI datasea_ocean_out.nc datasea_ocean_out_filled.nc
+rm -f datasea_ocean_out.nc
+
+# Merge special remapped variables + land fraction (renamed back to LSM)
+ncks -h -A dataland_out.nc era5_final.nc
+ncks -h -A datasea_ocean_out_filled.nc era5_final.nc
+ncks -h -A -v FR_LAND LSM_out.nc era5_final.nc
+ncrename -h -v FR_LAND,LSM era5_final.nc
+rm -f LSM_out.nc dataland_out.nc datasea_ocean_out_filled.nc
+
+# ---------------------------------------------------------------------
+# 4) Soil moisture index conversion (ERA5 SWVL -> ICON-style SMIL)
+# ---------------------------------------------------------------------
+# Extract soil moisture and soil type
+ncks -h -v SMIL1,SMIL2,SMIL3,SMIL4,SLT data_in.nc swvl.nc
+rm -f data_in.nc
+
+# IFS soil constants (documented in ERA5 data documentation)
+wiltingp=(0 0.059 0.151 0.133 0.279 0.335 0.267 0.151)
+fieldcap=(0 0.244 0.347 0.383 0.448 0.541 0.663 0.347)
+smi_equation=""
+for ilev in {1..4}; do
+  smi_equation="${smi_equation}SMIL${ilev}=(SMIL${ilev}-${wiltingp[1]})/(${fieldcap[1]}-${wiltingp[1]})*(SLT==1)"
+  for ist in {2..7}; do
+    smi_equation="${smi_equation}+(SMIL${ilev}-${wiltingp[$ist]})/(${fieldcap[$ist]}-${wiltingp[$ist]})*(SLT==${ist})"
+  done
+  smi_equation="${smi_equation};"
+done
+cdo expr,"${smi_equation}" swvl.nc smil_in.nc
+rm -f swvl.nc
+
+# Remap SMIL to triangular grid and overwrite
+cdo -s remapdis,triangular-grid.nc smil_in.nc smil_out.nc
+ncks -A -v SMIL1,SMIL2,SMIL3,SMIL4 smil_out.nc era5_final.nc
+rm -f smil_in.nc smil_out.nc
+
+# ---------------------------------------------------------------------
+# 5) Create LNPS (log surface pressure) and finalize dimensions
+# ---------------------------------------------------------------------
+cdo expr,'LNPS=ln(PS);' era5_final.nc tmp.nc
+ncks -A -v LNPS tmp.nc era5_final.nc
+rm -f tmp.nc
+
+# ICON conventions: rename dimensions
+ncrename -h -d cell,ncells era5_final.nc
+ncrename -h -d nv,vertices era5_final.nc
+
+# Write final IC file to the exact path ICON will read later
+mkdir -p "$(dirname "$OUTFILE")"
+ncks era5_final.nc "$OUTFILE"
+
+# cleanup
+rm -f era5_final.nc triangular-grid.nc
+echo "Wrote ICON initial condition file: $OUTFILE"