![\"pism-terra\"](\"_static/pism_logo_transp.png\"){kind=logo}
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cell-0",
+ "metadata": {},
+ "source": [
+ "# PISM-Cloud — interactive app\n",
+ "\n",
+ "Run this notebook with **Voila** to get a button-driven web UI (no need to run cells by hand):\n",
+ "\n",
+ "```\n",
+ "voila notebooks/pism_cloud_app.ipynb --strip_sources=True\n",
+ "```\n",
+ "\n",
+ "It wraps the same workflow as `pism_cloud.ipynb`: connect with your Earthdata (EDL) login,\n",
+ "upload a PISM config / template / (optional) UQ file, submit a *prepare* job, watch it,\n",
+ "then find the generated run scripts and *execute* them. AWS credentials come from your\n",
+ "local environment (the default boto3 chain / `~/.aws`)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cell-1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from copy import deepcopy\n",
+ "\n",
+ "import boto3\n",
+ "import s3fs\n",
+ "import hyp3_sdk as sdk\n",
+ "import ipywidgets as widgets\n",
+ "from IPython.display import display\n",
+ "\n",
+ "# Single shared session state (single kernel / single user).\n",
+ "state = {\n",
+ " \"client\": None,\n",
+ " \"jobs\": None,\n",
+ " \"pism_config\": None,\n",
+ " \"pism_template\": None,\n",
+ " \"pism_uq\": \"none\",\n",
+ "}\n",
+ "\n",
+ "_s3 = boto3.client(\"s3\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cell-2",
+ "metadata": {},
+ "source": [
+ "## 1. Connect to PISM-Cloud\n",
+ "\n",
+ "Enter your [Earthdata Login](https://urs.earthdata.nasa.gov/) credentials and click **Connect**."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cell-3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_cloud_server = \"https://pism-cloud-test.asf.alaska.edu\"\n",
+ "production_cloud_server = \"https://pism-cloud.asf.alaska.edu\"\n",
+ "\n",
+ "# Drop down menu to choose between the PISM Cloud Production and PISM Cloud Test\n",
+ "# servers. The default is the production Cloud.\n",
+ "server_w = widgets.Dropdown(\n",
+ " options=[\n",
+ " (\"PISM Cloud Production\", production_cloud_server),\n",
+ " (\"PISM Cloud Test\", test_cloud_server),\n",
+ " ],\n",
+ " value=production_cloud_server,\n",
+ " description=\"Server:\",\n",
+ " layout=widgets.Layout(width=\"520px\"),\n",
+ ")\n",
+ "user_w = widgets.Text(description=\"EDL user:\")\n",
+ "pass_w = widgets.Password(description=\"EDL pass:\")\n",
+ "connect_btn = widgets.Button(description=\"Connect\", button_style=\"primary\", icon=\"plug\")\n",
+ "connect_out = widgets.Output()\n",
+ "\n",
+ "\n",
+ "def _connect(_):\n",
+ " connect_out.clear_output()\n",
+ " with connect_out:\n",
+ " try:\n",
+ " client = sdk.HyP3(server_w.value, username=user_w.value, password=pass_w.value)\n",
+ " state[\"client\"] = client\n",
+ " print(f\"\\u2705 Connected. Credits: {client.check_credits()}\")\n",
+ " except Exception as exc: # noqa: BLE001\n",
+ " print(f\"\\u274c Connection failed: {exc}\")\n",
+ "\n",
+ "\n",
+ "connect_btn.on_click(_connect)\n",
+ "display(widgets.VBox([server_w, user_w, pass_w, connect_btn, connect_out]))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cell-4",
+ "metadata": {},
+ "source": [
+ "## 2. Glacier & metadata"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cell-5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rgi_w = widgets.Text(value=\"RGI2000-v7.0-C-01-04374\", description=\"RGI ID:\",\n",
+ " layout=widgets.Layout(width=\"420px\"))\n",
+ "name_w = widgets.Text(value=\"summer_school\", description=\"Name:\")\n",
+ "project_w = widgets.Text(value=\"test_inverse_fridays\", description=\"Project:\")\n",
+ "bucket_w = widgets.Text(value=\"pism-cloud-data\", description=\"Bucket:\")\n",
+ "ntasks_w = widgets.IntText(value=32, description=\"ntasks:\")\n",
+ "display(widgets.VBox([rgi_w, name_w, project_w, bucket_w, ntasks_w]))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cell-6",
+ "metadata": {},
+ "source": [
+ "## 3. Upload input files\n",
+ "\n",
+ "Upload the PISM `config` (`.toml`) and `template` (`.j2`); the `UQ` file (`.toml`) is optional.\n",
+ "Each file is shipped to `s3://{bucket}/glacier/{name}/{kind}/`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cell-7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def _make_uploader(kind: str, state_key: str, accept: str = \"\") -> widgets.VBox:\n",
+ " header = widgets.HTML(value=f\"Upload {kind} file\")\n",
+ " picker = widgets.FileUpload(accept=accept, multiple=False, description=f\"Browse {kind}…\")\n",
+ " status = widgets.HTML(value=\"no file uploaded yet\")\n",
+ "\n",
+ " def _on_change(_change):\n",
+ " if not picker.value:\n",
+ " return\n",
+ " # ipywidgets >= 8 returns a tuple of dicts; older versions return a dict.\n",
+ " item = picker.value[0] if isinstance(picker.value, (list, tuple)) else next(iter(picker.value.values()))\n",
+ " filename = item.get(\"name\") or item[\"metadata\"][\"name\"]\n",
+ " raw = item[\"content\"] if isinstance(item.get(\"content\"), bytes) else bytes(item[\"content\"])\n",
+ " key = f\"glacier/{name_w.value}/{kind}/{filename}\"\n",
+ " try:\n",
+ " _s3.put_object(Bucket=bucket_w.value, Key=key, Body=raw)\n",
+ " except Exception as exc: # noqa: BLE001\n",
+ " status.value = f\"❌ upload failed: {exc}\"\n",
+ " return\n",
+ " uri = f\"s3://{bucket_w.value}/{key}\"\n",
+ " state[state_key] = uri\n",
+ " status.value = f\"✅ uploaded → {uri}\"\n",
+ "\n",
+ " picker.observe(_on_change, names=\"value\")\n",
+ " return widgets.VBox([header, picker, status])\n",
+ "\n",
+ "\n",
+ "display(\n",
+ " widgets.VBox(\n",
+ " [\n",
+ " _make_uploader(\"config\", \"pism_config\", accept=\".toml\"),\n",
+ " _make_uploader(\"template\", \"pism_template\", accept=\".j2,.jinja,.jinja2\"),\n",
+ " _make_uploader(\"uq\", \"pism_uq\", accept=\".toml\"),\n",
+ " ]\n",
+ " )\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cell-8",
+ "metadata": {},
+ "source": [
+ "## 4. Run type & submit the prepare job"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cell-9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "runtype_w = widgets.RadioButtons(options=[\"forward\", \"inverse\"], description=\"Run type:\")\n",
+ "submit_btn = widgets.Button(description=\"Submit prepare job\", button_style=\"primary\", icon=\"rocket\")\n",
+ "submit_out = widgets.Output()\n",
+ "\n",
+ "_JOB_TYPE = {\"forward\": \"PISM_TERRA_RUN_FORWARD\", \"inverse\": \"PISM_TERRA_RUN_INVERSE\"}\n",
+ "\n",
+ "\n",
+ "def _submit_prepare(_):\n",
+ " submit_out.clear_output()\n",
+ " with submit_out:\n",
+ " if state[\"client\"] is None:\n",
+ " print(\"❌ Not connected — click Connect in step 1 first.\")\n",
+ " return\n",
+ " if not state[\"pism_config\"] or not state[\"pism_template\"]:\n",
+ " print(\"❌ Upload a config (.toml) and a template (.j2) first.\")\n",
+ " return\n",
+ " # Literal {name}/{job_id} are filled in by the PISM-Cloud backend.\n",
+ " bucket_prefix = f\"{{name}}/{project_w.value}/{{job_id}}\"\n",
+ " pism_job = {\n",
+ " \"job_type\": _JOB_TYPE[runtype_w.value],\n",
+ " \"name\": name_w.value,\n",
+ " \"bucket\": bucket_w.value,\n",
+ " \"bucket_prefix\": bucket_prefix,\n",
+ " \"job_parameters\": {\n",
+ " \"rgi_id\": rgi_w.value,\n",
+ " \"pism_config\": state[\"pism_config\"],\n",
+ " \"run_template\": state[\"pism_template\"],\n",
+ " \"uq_config\": state[\"pism_uq\"],\n",
+ " \"ntasks\": ntasks_w.value,\n",
+ " },\n",
+ " }\n",
+ " try:\n",
+ " jobs = state[\"client\"].submit_prepared_jobs(pism_job)\n",
+ " except Exception as exc: # noqa: BLE001\n",
+ " print(f\"❌ Submit failed: {exc}\")\n",
+ " return\n",
+ " state[\"jobs\"] = jobs\n",
+ " print(f\"✅ Submitted {len(jobs)} job(s):\")\n",
+ " for j in jobs:\n",
+ " print(f\" {j.job_id} {j.job_type} {j.status_code}\")\n",
+ "\n",
+ "\n",
+ "submit_btn.on_click(_submit_prepare)\n",
+ "display(widgets.VBox([runtype_w, submit_btn, submit_out]))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cell-10",
+ "metadata": {},
+ "source": [
+ "## 5. Job status\n",
+ "\n",
+ "Click **Refresh status** to poll once (non-blocking), or tick auto-refresh to poll every 30 s."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cell-11",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import threading\n",
+ "\n",
+ "refresh_btn = widgets.Button(description=\"Refresh status\", icon=\"refresh\")\n",
+ "autorefresh_w = widgets.Checkbox(value=False, description=\"Auto-refresh (30s)\")\n",
+ "status_out = widgets.Output()\n",
+ "_timer = {\"t\": None}\n",
+ "\n",
+ "\n",
+ "def _render_status():\n",
+ " status_out.clear_output()\n",
+ " with status_out:\n",
+ " jobs = state[\"jobs\"]\n",
+ " if not jobs:\n",
+ " print(\"No jobs submitted yet.\")\n",
+ " return\n",
+ " try:\n",
+ " jobs = state[\"client\"].refresh(jobs)\n",
+ " except Exception as exc: # noqa: BLE001\n",
+ " print(f\"❌ Refresh failed: {exc}\")\n",
+ " return\n",
+ " state[\"jobs\"] = jobs\n",
+ " for j in jobs:\n",
+ " print(f\" {j.job_id} {j.job_type} {j.status_code}\")\n",
+ "\n",
+ "\n",
+ "def _refresh(_):\n",
+ " _render_status()\n",
+ "\n",
+ "\n",
+ "def _schedule():\n",
+ " if not autorefresh_w.value:\n",
+ " return\n",
+ " _render_status()\n",
+ " timer = threading.Timer(30.0, _schedule)\n",
+ " timer.daemon = True\n",
+ " _timer[\"t\"] = timer\n",
+ " timer.start()\n",
+ "\n",
+ "\n",
+ "def _toggle_auto(change):\n",
+ " if change[\"new\"]:\n",
+ " _schedule()\n",
+ " elif _timer[\"t\"] is not None:\n",
+ " _timer[\"t\"].cancel()\n",
+ "\n",
+ "\n",
+ "refresh_btn.on_click(_refresh)\n",
+ "autorefresh_w.observe(_toggle_auto, names=\"value\")\n",
+ "display(widgets.VBox([widgets.HBox([refresh_btn, autorefresh_w]), status_out]))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cell-12",
+ "metadata": {},
+ "source": [
+ "## 6. Execute the simulations\n",
+ "\n",
+ "Once the prepare job(s) have **succeeded**, this finds the generated run scripts in S3 and\n",
+ "submits one `PISM_TERRA_EXECUTE` job per script. Track them with **Refresh status** above."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cell-13",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_run_scripts(job) -> list[str]:\n",
+ " fs = s3fs.S3FileSystem(anon=True)\n",
+ " files = fs.ls(f\"{bucket_w.value}/{job.name}/{project_w.value}/{job.job_id}/{rgi_w.value}/run_scripts/\")\n",
+ " return [f\"s3://{file}\" for file in files]\n",
+ "\n",
+ "\n",
+ "execute_btn = widgets.Button(description=\"Find scripts & execute\", button_style=\"primary\", icon=\"play\")\n",
+ "execute_out = widgets.Output()\n",
+ "\n",
+ "\n",
+ "def _execute(_):\n",
+ " execute_out.clear_output()\n",
+ " with execute_out:\n",
+ " if state[\"client\"] is None:\n",
+ " print(\"❌ Not connected.\")\n",
+ " return\n",
+ " jobs = state[\"jobs\"]\n",
+ " if not jobs:\n",
+ " print(\"❌ No prepare jobs found — submit them in step 4 and let them finish first.\")\n",
+ " return\n",
+ " execute_template = {\n",
+ " \"name\": name_w.value,\n",
+ " \"bucket\": bucket_w.value,\n",
+ " \"bucket_prefix\": f\"{{name}}/{project_w.value}/{{job_id}}\",\n",
+ " \"job_type\": \"PISM_TERRA_EXECUTE\",\n",
+ " \"job_parameters\": {},\n",
+ " }\n",
+ " prepared = []\n",
+ " for job in jobs:\n",
+ " try:\n",
+ " scripts = get_run_scripts(job)\n",
+ " except Exception as exc: # noqa: BLE001\n",
+ " print(f\"⚠️ could not list run scripts for {job.job_id}: {exc}\")\n",
+ " continue\n",
+ " for script in scripts:\n",
+ " job_dict = deepcopy(execute_template)\n",
+ " job_dict[\"name\"] = job.name\n",
+ " job_dict[\"job_parameters\"][\"run_script\"] = script\n",
+ " prepared.append(job_dict)\n",
+ " if not prepared:\n",
+ " print(\"No run scripts found yet (prepare jobs may still be running).\")\n",
+ " return\n",
+ " try:\n",
+ " exec_jobs = state[\"client\"].submit_prepared_jobs(prepared)\n",
+ " except Exception as exc: # noqa: BLE001\n",
+ " print(f\"❌ Execute submit failed: {exc}\")\n",
+ " return\n",
+ " state[\"jobs\"] = exec_jobs\n",
+ " print(f\"✅ Submitted {len(exec_jobs)} execute job(s). Track them with Refresh status above.\")\n",
+ " for j in exec_jobs:\n",
+ " print(f\" {j.job_id} {j.status_code}\")\n",
+ "\n",
+ "\n",
+ "execute_btn.on_click(_execute)\n",
+ "display(widgets.VBox([execute_btn, execute_out]))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "30f34af3-552f-4395-9f4d-bc2fa93e8673",
+ "metadata": {},
+ "source": [
+ "## Analysis\n",
+ "\n",
+ "Go to [Open Science Lab](https://opensciencelab.asf.alaska.edu/) to analyze the results, or download them."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "81b66353-d130-4b23-b083-131072adaab9",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebooks/s4f-grids.ipynb b/notebooks/s4f-grids.ipynb
new file mode 100644
index 0000000..1edc57f
--- /dev/null
+++ b/notebooks/s4f-grids.ipynb
@@ -0,0 +1,718 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "b17c8dfe-15d5-4c30-a621-14f165d6aec9",
+ "metadata": {},
+ "source": [
+ "# Snow4Flow Glacier Grids\n",
+ "\n",
+ "*Andy Aschwanden, April 2026*\n",
+ "\n",
+ "Here are some thoughts on S4F Glacier Grids:\n",
+ "\n",
+ " - Grids, CRS and associated properties should be chosen by the downstream users (the modeling community) by data collectors. This reduces the amount of reprojections needed.\n",
+ " - point data can be in EPSG:4326\n",
+ " - Ellipsoid vs Geoid: For modeling purposes we need sea level to be at 0m. Removing or adding geoids/ellipsoids are a pain and great care is needed to avoid ice thicknesses < 0."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "85e59bb2-45ee-488c-a05b-9470d068318e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip install boto3 cartopy geopandas matplotlib numpy pandas pathlib rioxarray shapely tqdm"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fd340d28-fccd-43bb-adbf-40ef5ed322d4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pathlib import Path\n",
+ "from urllib.parse import urlparse\n",
+ "\n",
+ "import boto3\n",
+ "from boto3.s3.transfer import TransferConfig\n",
+ "from botocore.config import Config\n",
+ "import cartopy.crs as ccrs \n",
+ "\n",
+ "import geopandas as gpd\n",
+ "import numpy as np\n",
+ "import matplotlib.pylab as plt\n",
+ "import matplotlib as mpl\n",
+ "from matplotlib.patches import FancyArrow\n",
+ "import pandas as pd\n",
+ "import rioxarray\n",
+ "import shapely\n",
+ "from shapely.geometry import Point, box\n",
+ "from tqdm import tqdm\n",
+ "import xarray as xr"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "60c56e73-77a0-45fa-a97e-2b719e6982a9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "proj = ccrs.PlateCarree()\n",
+ "\n",
+ "fontsize = 6\n",
+ "rc_params = {\n",
+ " \"axes.linewidth\": 0.15,\n",
+ " \"xtick.major.size\": 2.0,\n",
+ " \"xtick.major.width\": 0.15,\n",
+ " \"ytick.major.size\": 2.0,\n",
+ " \"ytick.major.width\": 0.15,\n",
+ " \"hatch.linewidth\": 0.15,\n",
+ " \"font.size\": fontsize,\n",
+ " \"font.family\": \"DejaVu Sans\",\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dd160caa-fbe7-4072-9a68-5a8eeccd1aad",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def north_arrow(ax, x=0.95, y=0.15, arrow_length=0.1, \n",
+ " label=\"N\", fontsize=14, lw=2, color=\"black\"):\n",
+ " \"\"\"\n",
+ " Draw a true-north arrow at axes-fraction (x, y). \n",
+ " Works for any Cartopy projection. The arrow points along the local \n",
+ " meridian at the anchor's geographic location. \n",
+ " \"\"\"\n",
+ " proj = ax.projection \n",
+ " pc = ccrs.PlateCarree() \n",
+ " \n",
+ " # 1. anchor (axes fraction) -> display -> data (projection) -> lon/lat \n",
+ " disp = ax.transAxes.transform((x, y))\n",
+ " px, py = ax.transData.inverted().transform(disp) \n",
+ " lon, lat = pc.transform_point(px, py, proj)\n",
+ " \n",
+ " # 2. a point a small step north in lon/lat, back into projection coords \n",
+ " dlat = 0.5 # degrees; small enough to be \"local\"\n",
+ " px2, py2 = proj.transform_point(lon, lat + dlat, pc) \n",
+ " \n",
+ " # 3. direction in *display* coords so length is visually consistent \n",
+ " d0 = ax.transData.transform((px, py)) \n",
+ " d1 = ax.transData.transform((px2, py2)) \n",
+ " dx, dy = d1 - d0\n",
+ " angle = np.arctan2(dy, dx) \n",
+ " \n",
+ " # 4. draw arrow + label in axes-fraction space, rotated to that angle \n",
+ " ux, uy = np.cos(angle), np.sin(angle) \n",
+ " # convert arrow_length (axes fraction) to a vector in axes coords \n",
+ " head = (x + ux * arrow_length, y + uy * arrow_length) \n",
+ " tail = (x - ux * arrow_length * 0.2, y - uy * arrow_length * 0.2) \n",
+ " \n",
+ " ax.annotate( \n",
+ " \"\", xy=head, xytext=tail, \n",
+ " xycoords=\"axes fraction\", textcoords=\"axes fraction\",\n",
+ " arrowprops=dict(arrowstyle=\"-|>\", lw=lw, color=color, \n",
+ " mutation_scale=20), \n",
+ " ) \n",
+ " ax.text( \n",
+ " head[0] + ux * 0.02, head[1] + uy * 0.02, label,\n",
+ " transform=ax.transAxes, \n",
+ " ha=\"center\", va=\"center\",\n",
+ " fontsize=fontsize, fontweight=\"bold\", color=color, \n",
+ " ) \n",
+ "\n",
+ "def download_from_s3(s3_uri: str, dest: str | Path) -> Path:\n",
+ " \"\"\"\n",
+ " Download a file from AWS S3.\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " s3_uri : str\n",
+ " URI of S3 object to download.\n",
+ " dest : str or Path\n",
+ " Path to the downloaded file.\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " Path\n",
+ " Path to the downloaded file.\n",
+ " \"\"\"\n",
+ " dest = Path(dest)\n",
+ "\n",
+ " parsed_url = urlparse(s3_uri)\n",
+ " bucket = parsed_url.netloc\n",
+ " prefix = parsed_url.path.lstrip(\"/\")\n",
+ "\n",
+ " s3 = boto3.client(\"s3\")\n",
+ " head = s3.head_object(Bucket=bucket, Key=prefix)\n",
+ " total_size = head[\"ContentLength\"]\n",
+ "\n",
+ " with tqdm(total=total_size, unit=\"B\", unit_scale=True, desc=dest.name) as pbar:\n",
+ " s3.download_file(bucket, prefix, str(dest), Callback=pbar.update)\n",
+ "\n",
+ " return dest\n",
+ "\n",
+ "def get_glacier_from_rgi_id(rgi: gpd.GeoDataFrame | str | Path, rgi_id: str) -> gpd.GeoDataFrame:\n",
+ " \"\"\"\n",
+ " Return the row in the GeoDataFrame matching the given RGI ID.\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " rgi : geopandas.GeoDataFrame\n",
+ " GeoDataFrame containing glacier data.\n",
+ " rgi_id : str\n",
+ " RGI identifier to look up.\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " geopandas.GeoSeries\n",
+ " The matching row.\n",
+ " \"\"\"\n",
+ " if isinstance(rgi, str | Path):\n",
+ " rgi = gpd.read_file(rgi)\n",
+ "\n",
+ " glacier = rgi[rgi[\"rgi_id\"] == rgi_id]\n",
+ " return glacier\n",
+ "\n",
+ "def create_domain(\n",
+ " x_bnds: list | np.ndarray,\n",
+ " y_bnds: list | np.ndarray,\n",
+ " resolution: float | None = None,\n",
+ " x_dim: str = \"x\",\n",
+ " y_dim: str = \"y\",\n",
+ " crs: str = \"EPSG:3413\",\n",
+ ") -> xr.Dataset:\n",
+ " \"\"\"\n",
+ " Create an xarray.Dataset representing a domain with specified x and y boundaries.\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " x_bnds : list or numpy.ndarray\n",
+ " A list or array containing the minimum and maximum x-coordinate boundaries.\n",
+ " y_bnds : list or numpy.ndarray\n",
+ " A list or array containing the minimum and maximum y-coordinate boundaries.\n",
+ " resolution : float or None, optional\n",
+ " The resolution of the grid, by default None.\n",
+ " x_dim : str, optional\n",
+ " The name of the x dimension, by default \"x\".\n",
+ " y_dim : str, optional\n",
+ " The name of the y dimension, by default \"y\".\n",
+ " crs : str, optional\n",
+ " The coordinate reference system (CRS) for the domain, by default \"EPSG:3413\".\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " xarray.Dataset\n",
+ " An xarray.Dataset containing the domain information, including coordinates,\n",
+ " boundary data, and mapping attributes.\n",
+ "\n",
+ " Notes\n",
+ " -----\n",
+ " The dataset includes:\n",
+ " - `x` and `y` coordinates with associated metadata.\n",
+ " - A `mapping` DataArray with polar stereographic projection attributes.\n",
+ " - A `domain` DataArray with a reference to the `mapping`.\n",
+ " - `x_bnds` and `y_bnds` DataArrays representing the boundaries of the domain.\n",
+ "\n",
+ " Examples\n",
+ " --------\n",
+ " >>> x_bnds = [0, 1000]\n",
+ " >>> y_bnds = [0, 2000]\n",
+ " >>> ds = create_domain(x_bnds, y_bnds)\n",
+ " >>> print(ds)\n",
+ " \"\"\"\n",
+ "\n",
+ " if resolution is not None:\n",
+ " # np.arange(start, stop) includes start but not stop\n",
+ " xb = np.arange(x_bnds[0], x_bnds[1] + resolution, resolution)\n",
+ " yb = np.arange(y_bnds[0], y_bnds[1] + resolution, resolution)\n",
+ " x = np.arange(x_bnds[0] + resolution / 2, x_bnds[1] + resolution - resolution / 2, resolution)\n",
+ " y = np.arange(y_bnds[0] + resolution / 2, y_bnds[1] + resolution - resolution / 2, resolution)\n",
+ " x_bounds = np.stack([xb[:-1], xb[1:]]).T\n",
+ " y_bounds = np.stack([yb[:-1], yb[1:]]).T\n",
+ " else:\n",
+ " x = np.array([0])\n",
+ " y = np.array([0])\n",
+ " x_bounds = np.array([[x_bnds[0], x_bnds[1]]])\n",
+ " y_bounds = np.array([[y_bnds[0], y_bnds[1]]])\n",
+ "\n",
+ " x_bnds_dim = f\"{x_dim}_bnds\"\n",
+ " y_bnds_dim = f\"{y_dim}_bnds\"\n",
+ " coords = {\n",
+ " x_dim: (\n",
+ " [x_dim],\n",
+ " x,\n",
+ " {\n",
+ " \"axis\": x_dim.upper(),\n",
+ " \"bounds\": x_bnds_dim,\n",
+ " },\n",
+ " ),\n",
+ " y_dim: (\n",
+ " [y_dim],\n",
+ " y,\n",
+ " {\n",
+ " \"axis\": y_dim.upper(),\n",
+ " \"bounds\": y_bnds_dim,\n",
+ " },\n",
+ " ),\n",
+ " }\n",
+ " ds = xr.Dataset(\n",
+ " {\n",
+ " \"domain\": xr.DataArray(\n",
+ " data=0,\n",
+ " dims=[y_dim, x_dim],\n",
+ " coords={x_dim: coords[x_dim], y_dim: coords[y_dim]},\n",
+ " attrs={\n",
+ " \"dimensions\": f\"{x_dim} {y_dim}\",\n",
+ " },\n",
+ " ),\n",
+ " x_bnds_dim: xr.DataArray(\n",
+ " data=x_bounds,\n",
+ " dims=[x_dim, \"nv2\"],\n",
+ " coords={x_dim: coords[x_dim]},\n",
+ " ),\n",
+ " y_bnds_dim: xr.DataArray(\n",
+ " data=y_bounds,\n",
+ " dims=[y_dim, \"nv2\"],\n",
+ " coords={y_dim: coords[y_dim]},\n",
+ " ),\n",
+ " },\n",
+ " attrs={\"Conventions\": \"CF-1.8\"},\n",
+ " ).rio.set_spatial_dims(x_dim=x_dim, y_dim=y_dim)\n",
+ " ds.rio.write_crs(crs, inplace=True).rio.write_coordinate_system(inplace=True)\n",
+ " for var in list(ds.data_vars) + list(ds.coords):\n",
+ " ds[var].encoding[\"_FillValue\"] = None\n",
+ "\n",
+ " return ds\n",
+ "\n",
+ "def grid_points_from_dataset(ds: xr.Dataset) -> gpd.GeoDataFrame:\n",
+ " \"\"\"\n",
+ " Build a GeoDataFrame of grid cell centers from a domain dataset.\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " ds : xarray.Dataset\n",
+ " Dataset with ``x`` and ``y`` coordinates (cell centers) and a\n",
+ " ``spatial_ref`` variable carrying CRS information in ``crs_wkt``.\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " geopandas.GeoDataFrame\n",
+ " One Point geometry per cell center, in the dataset's CRS.\n",
+ " \"\"\"\n",
+ " xs, ys = np.meshgrid(ds.x.values, ds.y.values)\n",
+ " points = shapely.points(xs.ravel(), ys.ravel())\n",
+ " return gpd.GeoDataFrame(geometry=points, crs=ds.spatial_ref.attrs.get(\"crs_wkt\"))\n",
+ "\n",
+ "\n",
+ "def grid_cells_from_dataset(ds: xr.Dataset) -> gpd.GeoDataFrame:\n",
+ " \"\"\"\n",
+ " Build a GeoDataFrame of grid cell polygons from a domain dataset.\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " ds : xarray.Dataset\n",
+ " Dataset with ``x_bnds`` and ``y_bnds`` cell-edge bounds, a ``domain``\n",
+ " variable, and a ``spatial_ref`` variable carrying CRS information in\n",
+ " ``crs_wkt``.\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " geopandas.GeoDataFrame\n",
+ " One rectangular Polygon per grid cell, with a ``domain`` column\n",
+ " copied from ``ds.domain`` and the dataset's CRS.\n",
+ " \"\"\"\n",
+ " x0, x1 = ds.x_bnds.values[:, 0], ds.x_bnds.values[:, 1]\n",
+ " y0, y1 = ds.y_bnds.values[:, 0], ds.y_bnds.values[:, 1]\n",
+ " X0, Y0 = np.meshgrid(x0, y0)\n",
+ " X1, Y1 = np.meshgrid(x1, y1)\n",
+ " polys = shapely.box(X0.ravel(), Y0.ravel(), X1.ravel(), Y1.ravel())\n",
+ " return gpd.GeoDataFrame(\n",
+ " {\"domain\": ds.domain.values.flatten()},\n",
+ " geometry=polys,\n",
+ " crs=ds.spatial_ref.attrs.get(\"crs_wkt\"),\n",
+ " )\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bb8f8401-2ea9-4d7e-aa5e-e2ffcdcd7049",
+ "metadata": {},
+ "source": [
+ "## Download RGI v7\n",
+ "\n",
+ "This is the same as on NSIDC but in GPKG format\n",
+ "\n",
+ "For all modeling purposes we will be using the glacier complexes `C` of RGI not the indiviual glaciers `G`. The reason is related to how lateral boundary conditions are applied, using complexes avoids the complexity of lateral boundaries with ice thickness"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a67f0e2f-fa52-483f-9713-7165e3edf21b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "bucket = \"pism-cloud-data\"\n",
+ "prefix = \"s4f\"\n",
+ "\n",
+ "print(\"RGI Glacier Complexes (C)\")\n",
+ "rgi_c_file = \"rgi_c.gpkg\"\n",
+ "rgi_c_s3_uri = f\"\"\"s3://{bucket}/{prefix}/rgi/{rgi_c_file}\"\"\"\n",
+ "rgi_c_local = Path(\".\") / rgi_c_file\n",
+ "\n",
+ "if not rgi_c_local.exists():\n",
+ " print(f\"Downloading {rgi_c_s3_uri} -> {rgi_c_local}\")\n",
+ " download_from_s3(rgi_c_s3_uri, rgi_c_local)\n",
+ "else:\n",
+ " print(f\"Using cached {rgi_c_local}\")\n",
+ "rgi_c = gpd.read_file(rgi_c_local)\n",
+ "\n",
+ "rgi_c_file = \"rgi_c.gpkg\"\n",
+ "\n",
+ "print(\"RGI Glaciers (G)\")\n",
+ "rgi_g_file = \"rgi_g.gpkg\"\n",
+ "rgi_g_s3_uri = f\"\"\"s3://{bucket}/{prefix}/rgi/{rgi_g_file}\"\"\"\n",
+ "rgi_g_local = Path(\".\") / rgi_g_file\n",
+ "\n",
+ "if not rgi_g_local.exists():\n",
+ " print(f\"Downloading {rgi_g_s3_uri} -> {rgi_g_local}\")\n",
+ " download_from_s3(rgi_g_s3_uri, rgi_g_local)\n",
+ "else:\n",
+ " print(f\"Using cached {rgi_g_local}\")\n",
+ "rgi_g = gpd.read_file(rgi_g_local)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "602cdba1-2420-40b3-833e-8e1e23861b63",
+ "metadata": {},
+ "source": [
+ "## Select a glacier\n",
+ "\n",
+ "Wrangell Glacier Complex\n",
+ "\n",
+ "rgi_id = \"RGI2000-v7.0-C-01-04374\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bc6c191f-c023-40ff-9faf-fab9369c9df4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rgi_id = \"RGI2000-v7.0-C-01-04374\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f6cc7075-797f-4122-94d6-ebf5b5deaab6",
+ "metadata": {},
+ "source": [
+ "## Option 1: EPSG:3413"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "08761bf0-09f0-449a-a826-bc0485f9b0bf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get glacier and plot in dst_crs\n",
+ "dst_crs = \"EPSG:3413\"\n",
+ "dst_proj = ccrs.NorthPolarStereo(central_longitude=-45, true_scale_latitude=70) \n",
+ "glacier = get_glacier_from_rgi_id(rgi_c, rgi_id)\n",
+ "glacier_projected = glacier.to_crs(dst_crs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b878fd17-ac24-410f-bcf6-be4ecc023398",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "buffer_dist = 5000.0\n",
+ "bounds = glacier_projected.buffer(buffer_dist).geometry.bounds"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f2b63f17-d560-49e1-b8bf-9769e65101c5",
+ "metadata": {},
+ "source": [
+ "### Grid Bounds and Resolution\n",
+ "\n",
+ "For ease of visualization, we set `dx=5000`, however the final grids are expected to be around `dx ~ 500m`. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d8f3d90b-476c-4c09-bf6e-4e91006f15b1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Round in projected meters to a dx=5000 m grid\n",
+ "dx = 5000\n",
+ "x_min = np.ceil((bounds.minx.item()) / dx) * dx\n",
+ "x_max = np.floor((bounds.maxx.item()) / dx) * dx\n",
+ "y_min = np.ceil((bounds.miny.item()) / dx) * dx\n",
+ "y_max = np.floor((bounds.maxy.item()) / dx) * dx"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "73f1e952-25e5-4bf9-8461-1bc54030588d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "target_grid = create_domain([x_min, x_max], [y_min, y_max], resolution=dx, crs=dst_crs)\n",
+ "gdf_cells = grid_cells_from_dataset(target_grid)\n",
+ "gdf_points = grid_points_from_dataset(target_grid)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "59b5ea8e-0674-4845-9639-96969fdd149b",
+ "metadata": {},
+ "source": [
+ "### The Glacier Grid in EPSG:3413\n",
+ "\n",
+ "Compared to `Alaska Albers` or `UTM`, Wrangell is rotated by 90 degrees."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "139c1d19-4c74-4370-b843-802125c30d96",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with mpl.rc_context(rc=rc_params): \n",
+ " fig, ax = plt.subplots(figsize=(8, 8), subplot_kw={\"projection\": dst_proj})\n",
+ " gdf_cells.boundary.plot(ax=ax, color=\"gray\", lw=0.3, transform=dst_proj) \n",
+ " gdf_points.plot(ax=ax, color=\"red\", markersize=0.5, transform=dst_proj)\n",
+ " glacier_projected.boundary.plot(ax=ax, transform=dst_proj)\n",
+ " north_arrow(ax, x=0.92, y=0.15, arrow_length=0.08)\n",
+ "\n",
+ "plt.show()\n",
+ "del fig"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "eb263e20-b372-4690-a6c7-dc63a18bb3f2",
+ "metadata": {},
+ "source": [
+ "## Option B: Local UTM Zone from RGI7"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "51f3b355-25d9-420c-ba3b-83741d016e71",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get glacier and plot in dst_crs\n",
+ "glacier = get_glacier_from_rgi_id(rgi_c, rgi_id)\n",
+ "glacier_series = glacier.iloc[0]\n",
+ "dst_crs = glacier_series[\"epsg\"]\n",
+ "dst_proj = ccrs.epsg(glacier_series[\"epsg_code\"])\n",
+ "glacier_projected = glacier.to_crs(dst_crs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "67de14cc-6c24-4647-a6dc-e64830592278",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "buffer_dist = 5000.0\n",
+ "bounds = glacier_projected.buffer(buffer_dist).geometry.bounds"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d0c52dd6-ee8b-48a3-8bdd-08d3dfac0f91",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Round in projected meters to a dx=1000 m grid\n",
+ "dx = 5000\n",
+ "x_min = np.ceil((bounds.minx.item()) / dx) * dx\n",
+ "x_max = np.floor((bounds.maxx.item()) / dx) * dx\n",
+ "y_min = np.ceil((bounds.miny.item()) / dx) * dx\n",
+ "y_max = np.floor((bounds.maxy.item()) / dx) * dx"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5d6409a6-267b-4157-b307-f7368aeee123",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "target_grid = create_domain([x_min, x_max], [y_min, y_max], resolution=dx, crs=dst_crs)\n",
+ "gdf_cells = grid_cells_from_dataset(target_grid)\n",
+ "gdf_points = grid_points_from_dataset(target_grid)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "16d367d8-0ef1-4b8d-8346-968c348c2544",
+ "metadata": {},
+ "source": [
+ "### The Glacier Grid in the local UTM"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "33cafa94-9cff-47de-9764-1bbeb0dc5c8a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with mpl.rc_context(rc=rc_params): \n",
+ " fig, ax = plt.subplots(figsize=(8, 8), subplot_kw={\"projection\": dst_proj})\n",
+ " gdf_cells.boundary.plot(ax=ax, color=\"gray\", lw=0.3) \n",
+ " gdf_points.plot(ax=ax, color=\"red\", markersize=0.5)\n",
+ " glacier_projected.boundary.plot(ax=ax)\n",
+ " north_arrow(ax, x=0.92, y=0.15, arrow_length=0.08)\n",
+ "\n",
+ "plt.show()\n",
+ "del fig"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3dbc7ae0-25f0-4151-b01f-b8b4cac1aef6",
+ "metadata": {},
+ "source": [
+ "## Option C: Albers Equal Area\n",
+ "\n",
+ "Equal Area projections minimize distortions while we can still have a more natural north-up orientation. For the three S4F regions, we could use:\n",
+ "\n",
+ " - Alaska/Yukon: Alaska Albers (EPSG:3338)\n",
+ " - Arctic Canada: Canada Albers Equal Area Conic: (ESRI:102001)\n",
+ " - Svalbard: Europe Equal Area 2001 (ESRI:102013)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "27c90b06-22d5-4c99-93a7-2af31ee57b46",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "s4f_regional_glaciers = {\"RGI2000-v7.0-C-01-04374\": \"EPSG:3338\",\n",
+ " \"RGI2000-v7.0-C-03-01936\": \"ESRI:102001\",\n",
+ " \"RGI2000-v7.0-C-07-00364\": \"ESRI:102013\"} "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "263169ff-6dba-46cd-89fe-6913909e94de",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def _ccrs_projection(crs_str: str) -> ccrs.Projection:\n",
+ " \"\"\"CRS string → Cartopy Projection usable for axes projection.\"\"\"\n",
+ " auth, _, code = crs_str.partition(\":\") \n",
+ " if auth.upper() == \"EPSG\": \n",
+ " return ccrs.epsg(code) # works for real EPSG, e.g. 3338 \n",
+ " \n",
+ " if crs_str == \"ESRI:102001\": # Canada Albers Equal Area Conic \n",
+ " return ccrs.AlbersEqualArea( \n",
+ " central_longitude=-96, central_latitude=40, \n",
+ " standard_parallels=(50, 70), \n",
+ " globe=ccrs.Globe(datum=\"NAD83\", ellipse=\"GRS80\"),\n",
+ " ) \n",
+ " if crs_str == \"ESRI:102013\": # Europe Albers Equal Area Conic\n",
+ " return ccrs.AlbersEqualArea( \n",
+ " central_longitude=10, central_latitude=30, \n",
+ " standard_parallels=(43, 62), \n",
+ " ) \n",
+ " raise ValueError(f\"Unsupported CRS: {crs_str}\")\n",
+ " \n",
+ "\n",
+ "\n",
+ "with mpl.rc_context(rc=rc_params): \n",
+ " fig = plt.figure(figsize=(8, 3)) \n",
+ " \n",
+ " for k, (rgi_id, dst_crs) in enumerate(s4f_regional_glaciers.items()): \n",
+ " proj = _ccrs_projection(dst_crs) \n",
+ " \n",
+ " glacier = get_glacier_from_rgi_id(rgi_c, rgi_id) \n",
+ " glacier_projected = glacier.to_crs(dst_crs)\n",
+ " \n",
+ " bounds = glacier_projected.buffer(5000.0).geometry.bounds \n",
+ " dx = 5000\n",
+ " x_min = np.ceil(bounds.minx.item() / dx) * dx \n",
+ " x_max = np.floor(bounds.maxx.item() / dx) * dx \n",
+ " y_min = np.ceil(bounds.miny.item() / dx) * dx \n",
+ " y_max = np.floor(bounds.maxy.item() / dx) * dx\n",
+ " \n",
+ " target_grid = create_domain([x_min, x_max], [y_min, y_max], resolution=dx, crs=dst_crs)\n",
+ " gdf_cells = grid_cells_from_dataset(target_grid) \n",
+ " gdf_points = grid_points_from_dataset(target_grid) \n",
+ " \n",
+ " ax = fig.add_subplot(1, 3, k + 1, projection=proj) \n",
+ " ax.set_xlim(x_min, x_max) \n",
+ " ax.set_ylim(y_min, y_max) \n",
+ " \n",
+ " gdf_cells.boundary.plot(ax=ax, color=\"gray\", lw=0.3, transform=proj) \n",
+ " gdf_points.plot(ax=ax, color=\"red\", markersize=0.5, transform=proj)\n",
+ " glacier_projected.boundary.plot(ax=ax, transform=proj) \n",
+ " north_arrow(ax, x=0.92, y=0.15, arrow_length=0.08)\n",
+ "\n",
+ " ax.set_title(dst_crs)\n",
+ " \n",
+ " fig.savefig(\"s4f_crs.png\", dpi=300)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "76f3eb19-92c3-46c4-bf1e-443df408c8ba",
+ "metadata": {},
+ "source": [
+ "## Recommendations\n",
+ "\n",
+ " - To minimize distortions near the pole, we recommend adopting Albers equal-area projections for S4F."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/pism_terra/__main__.py b/pism_terra/__main__.py
index ab1bd20..123e0ab 100644
--- a/pism_terra/__main__.py
+++ b/pism_terra/__main__.py
@@ -14,33 +14,39 @@ def main():
"""
PISM-TERRA entrypoint dispatcher for PISM-Cloud.
"""
+ # Derive the allowed processes from the package's actual console_scripts
+ # entries so this dispatcher stays in sync when scripts are renamed/added
+ # in pyproject.toml.
+ eps = entry_points(group="console_scripts")
+ pism_processes = sorted(
+ {ep.name for ep in eps if ep.name.startswith("pism-") or ep.name == "combine-crameri-colormaps"}
+ )
+
parser = argparse.ArgumentParser(prefix_chars="+", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"++process",
- choices=[
- "pism-glacier-stage",
- "pism-glacier-run",
- "pism-glacier-run-ensemble",
- "pism-glacier-execute",
- "pism-glacier-postprocess",
- "combine-crameri-colormaps",
- ],
+ choices=pism_processes,
default="pism-glacier-stage",
help="Select the console_script entrypoint to use", # as specified in `pyproject.toml`
)
args, unknowns = parser.parse_known_args()
- cds_api_url = os.environ.get("CDS_API_URL")
- cds_api_key = os.environ.get("CDS_API_KEY")
- if (cds_api_file := Path.home() / ".cdsapirc").exists():
- warnings.warn(
- "CDS API credentials provided in both environment variables and the `~/.cdsapirc` file. Preferring file."
- )
- else:
- cds_api_file.write_text(f"url: {cds_api_url}\nkey: {cds_api_key}\n")
+ # Hand off credentials to ecmwf-datastores-client. It accepts either env
+ # vars (ECMWF_DATASTORES_URL / ECMWF_DATASTORES_KEY) or ~/.ecmwfdatastoresrc.
+ # We accept either CDS_API_* (legacy) or ECMWF_DATASTORES_* and write the
+ # config file when the rc file isn't already present.
+ cds_url = os.environ.get("ECMWF_DATASTORES_URL") or os.environ.get("CDS_API_URL")
+ cds_key = os.environ.get("ECMWF_DATASTORES_KEY") or os.environ.get("CDS_API_KEY")
+ if (cds_rc_file := Path.home() / ".ecmwfdatastoresrc").exists():
+ if cds_url or cds_key:
+ warnings.warn(
+ "ECMWF Data Stores credentials provided in both environment variables "
+ "and the `~/.ecmwfdatastoresrc` file. Preferring file."
+ )
+ elif cds_url and cds_key:
+ cds_rc_file.write_text(f"url: {cds_url}\nkey: {cds_key}\n")
- eps = entry_points(group="console_scripts")
(process_entry_point,) = {process for process in eps if process.name == args.process}
sys.argv = [args.process, *unknowns]
diff --git a/pism_terra/aws.py b/pism_terra/aws.py
index 1e9f40b..0b02f6f 100644
--- a/pism_terra/aws.py
+++ b/pism_terra/aws.py
@@ -25,6 +25,7 @@
import hashlib
import logging
import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Iterable
from urllib.parse import urlparse
@@ -32,6 +33,7 @@
import boto3
from boto3.s3.transfer import TransferConfig
from botocore.config import Config
+from tqdm import tqdm
logger = logging.getLogger(__name__)
@@ -62,8 +64,16 @@ def download_from_s3(s3_uri: str, dest: str | Path) -> Path:
bucket = parsed_url.netloc
prefix = parsed_url.path.lstrip("/")
- s3 = boto3.client("s3")
- s3.download_file(bucket, prefix, str(dest))
+ # Look up the bucket's region once; boto3's default region resolution is
+ # often wrong on shared dev machines and S3 returns 404 (not 307) when the
+ # request hits the wrong endpoint.
+ bucket_region = boto3.client("s3").get_bucket_location(Bucket=bucket).get("LocationConstraint") or "us-west-2"
+ s3 = boto3.client("s3", region_name=bucket_region)
+ head = s3.head_object(Bucket=bucket, Key=prefix)
+ total_size = head["ContentLength"]
+
+ with tqdm(total=total_size, unit="B", unit_scale=True, desc=dest.name) as pbar:
+ s3.download_file(bucket, prefix, str(dest), Callback=pbar.update)
return dest
@@ -150,6 +160,7 @@ def s3_to_local(
exclude_keys: Iterable[str] = (),
dry_run: bool = False,
delete_extra: bool = False,
+ workers: int = 4,
max_concurrency: int = 8,
) -> None:
"""
@@ -170,8 +181,14 @@ def s3_to_local(
delete_extra : bool, default False
If True, delete local files under ``dest`` that are not present
under ``bucket/prefix``.
+ workers : int, default 4
+ Number of *files* downloaded in parallel (outer fan-out). Each file
+ transfer additionally uses up to ``max_concurrency`` threads
+ internally for multipart parts, so the maximum in-flight connection
+ count is roughly ``workers * max_concurrency``.
max_concurrency : int, default 8
- Maximum worker threads for concurrent transfers.
+ Maximum worker threads boto3 may use **inside one** multipart
+ download.
Raises
------
@@ -206,7 +223,11 @@ def s3_to_local(
s3_local_abs = set()
n_objects = 0
-
+ # First pass: walk the listing, decide which objects need a transfer.
+ # Per-file tqdm bytes bars don't compose well with parallel workers
+ # (they interleave on the terminal), so the second pass just shows a
+ # single overall files-completed bar.
+ to_download: list[tuple[str, Path]] = []
for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
for obj in page.get("Contents", []):
key = obj["Key"]
@@ -216,14 +237,50 @@ def s3_to_local(
rel = key[len(prefix) :] if prefix and key.startswith(prefix) else key
local_path = dest / rel.lstrip("/")
local_path.parent.mkdir(parents=True, exist_ok=True)
-
if _needs_download(local_path, obj["Size"], obj["ETag"]):
- print("DOWNLOAD", f"s3://{bucket}/{key}", "→", local_path)
- if not dry_run:
- s3.download_file(bucket, key, str(local_path), Config=txconf)
-
+ to_download.append((key, local_path))
s3_local_abs.add(str(local_path.resolve()))
+ # Second pass: download in parallel. boto3 clients are safe for concurrent
+ # ``download_file`` calls; ``workers`` outer × ``max_concurrency`` inner
+ # threads keeps the connection pool busy without saturating CPU.
+ def _fetch(key: str, local_path: Path) -> Path:
+ """
+ Download one S3 object to *local_path*.
+
+ Parameters
+ ----------
+ key : str
+ S3 object key under ``bucket`` to download.
+ local_path : pathlib.Path
+ Local destination path.
+
+ Returns
+ -------
+ pathlib.Path
+ ``local_path`` after the transfer completes.
+ """
+ logger.info("Downloading s3://%s/%s -> %s", bucket, key, local_path)
+ s3.download_file(bucket, key, str(local_path), Config=txconf)
+ return local_path
+
+ if dry_run:
+ for key, local_path in to_download:
+ logger.info("[dry-run] would download s3://%s/%s -> %s", bucket, key, local_path)
+ elif to_download:
+ with ThreadPoolExecutor(max_workers=workers) as executor:
+ future_to_key = {executor.submit(_fetch, k, lp): k for k, lp in to_download}
+ for future in tqdm(
+ as_completed(future_to_key),
+ total=len(future_to_key),
+ desc=f"s3://{bucket}/{prefix}",
+ unit="file",
+ ):
+ try:
+ future.result()
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ logger.error("Failed s3://%s/%s: %s", bucket, future_to_key[future], exc)
+
if n_objects == 0:
logger.warning(
"No objects found in s3://%s/%s — check that the bucket and prefix are correct.",
diff --git a/pism_terra/config.py b/pism_terra/config.py
index 8160c3b..e111b24 100644
--- a/pism_terra/config.py
+++ b/pism_terra/config.py
@@ -29,7 +29,7 @@
import scipy.stats as st
import toml
-from jinja2 import Environment, StrictUndefined
+from jinja2 import Environment
from pydantic import (
BaseModel,
ConfigDict,
@@ -38,8 +38,12 @@
model_validator,
)
+# Dependency-free (imports only the stdlib), so a top-level import here cannot
+# create a cycle back into this module.
+from pism_terra.ismip7.experiments import resolve_counter
+
# one Jinja environment for all renders
-_JINJA = Environment(undefined=StrictUndefined, autoescape=False)
+_JINJA = Environment(autoescape=False)
def load_config(path: str | Path) -> PismConfig:
@@ -244,10 +248,15 @@ class UQConfig(BaseModel):
----------
samples : int, default=1
Number of draws to use when generating ensemble samples. Must be > 0.
- mapping : str or None, optional
- Optional column name indicating a mapping key (e.g., to join against a
- lookup table of file paths). Not interpreted by validation; simply
- preserved for downstream use.
+ For ``method="factorial"`` this is the number of levels **per variable**
+ (total runs are ``samples ** n_variables``).
+ method : str or None, optional
+ Sampling strategy passed to :func:`pism_terra.sampling.generate_samples`.
+ ``None`` (default) or ``"lhs"`` selects Latin Hypercube; ``"factorial"``
+ selects the full-factorial grid.
+ mapping : dict or None, optional
+ Optional mapping (e.g., to join against a lookup table of file paths).
+ Not interpreted by validation; simply preserved for downstream use.
tree : dict[str, DistSpec]
Flat mapping from dotted variable names to validated :class:`DistSpec`
objects.
@@ -288,9 +297,49 @@ class UQConfig(BaseModel):
"""
samples: int = Field(default=1, gt=0)
+ method: str | None = Field(default=None)
mapping: dict | None = None
tree: dict[str, "DistSpec"] # values parsed as DistSpec after 'before' validator
+ @field_validator("method")
+ @classmethod
+ def _normalize_method(cls, v: Any) -> str | None:
+ """
+ Normalize and validate the sampling ``method`` name.
+
+ Parameters
+ ----------
+ v : Any
+ Raw method value (any case, may contain surrounding spaces) or
+ ``None``.
+
+ Returns
+ -------
+ str or None
+ Lower-cased, stripped method name, or ``None`` when unset (which
+ downstream treats as Latin Hypercube).
+
+ Raises
+ ------
+ ValueError
+ If the method is not a recognized Latin Hypercube or factorial alias.
+ """
+ if v is None:
+ return None
+ m = str(v).strip().lower()
+ allowed = {
+ "lhs",
+ "latin",
+ "latin_hypercube",
+ "latinhypercube",
+ "factorial",
+ "grid",
+ "full_factorial",
+ }
+ if m not in allowed:
+ raise ValueError(f"unknown sampling method '{v}'; use 'lhs' or 'factorial'")
+ return m
+
@staticmethod
def _is_leaf(node: Any) -> bool:
"""
@@ -402,6 +451,7 @@ def _flatten_input(cls, v: Any) -> dict[str, Any]:
# Pull top-level fields if present
samples = v.get("samples")
+ method = v.get("method")
mapping = v.get("mapping")
# Where the specs live: either under 'tree' or at top level
@@ -410,20 +460,25 @@ def _flatten_input(cls, v: Any) -> dict[str, Any]:
outv: dict[str, Any] = {"tree": {}}
if samples is not None:
outv["samples"] = samples
+ if method is not None:
+ outv["method"] = method
if mapping is not None:
outv["mapping"] = mapping
return outv
raw = dict(raw) # shallow copy so we can pop safely
- # Allow samples/mapping inside the raw block too
+ # Allow samples/method/mapping inside the raw block too
if samples is None and "samples" in raw:
samples = raw.pop("samples")
+ if method is None and "method" in raw:
+ method = raw.pop("method")
if mapping is None and "mapping" in raw:
mapping = raw.pop("mapping")
# Ensure these don't leak into tree
raw.pop("samples", None)
+ raw.pop("method", None)
raw.pop("mapping", None)
# If keys are already dotted (['a.b.c']), keep only dict-valued items
@@ -436,6 +491,8 @@ def _flatten_input(cls, v: Any) -> dict[str, Any]:
out: dict[str, Any] = {"tree": tree}
if samples is not None:
out["samples"] = samples
+ if method is not None:
+ out["method"] = method
if mapping is not None:
out["mapping"] = mapping
return out
@@ -579,95 +636,6 @@ def _normalize_dotted_keys(cls, v: Any) -> Any:
return out
-class RunConfig(BaseModel):
- """
- Execution settings for a PISM run.
-
- Provides executable/launcher options and a helper to export parameters
- for templating. String fields that contain Jinja expressions (e.g.,
- ``"mpirun -np {{ ntasks }}"``) are rendered using the model values.
-
- Attributes
- ----------
- mpi : str
- MPI launcher template, e.g., ``"mpirun -np {{ ntasks }}"``.
- Defaults to ``"mpirun"``.
- executable : str
- Path to the PISM executable, or command name. Defaults to ``"pism"``.
- ntasks : int
- Total number of MPI ranks. Must be >= 1.
-
- Notes
- -----
- The :meth:`as_params` method returns only non-empty fields and renders any
- string value containing Jinja delimiters ``{{ ... }}`` using the current
- field values (plus any extra context provided).
-
- Examples
- --------
- >>> rc = RunConfig(mpi="mpirun -np {{ ntasks }}", executable="/path/pism", ntasks=56)
- >>> rc.as_params()["mpi"]
- 'mpirun -np 56'
- """
-
- mpi: str = Field(default="mpirun")
- executable: str = Field(default="pism")
- ntasks: int = Field(ge=1)
- writer: str | None = None
-
- def as_params(self, **extra: Any) -> dict[str, Any]:
- """
- Export non-empty parameters and render templated strings.
-
- Any string field containing Jinja expressions is rendered using a
- context composed of the model's own values plus ``extra``.
-
- Parameters
- ----------
- **extra
- Additional key/value pairs to inject into the Jinja render
- context (these do not mutate the model).
-
- Returns
- -------
- dict of str to Any
- Dictionary of parameters suitable for template rendering.
- Fields with ``None``/unset/default values are omitted; templated
- strings (e.g., ``mpi``) are rendered to plain strings.
- """
- params = self.model_dump(exclude_none=True, exclude_unset=True, exclude_defaults=True)
- ctx = {**params, **extra}
-
- def _render(v: Any) -> Any:
- """
- Render templated strings using the current context.
-
- Parameters
- ----------
- v : Any
- Candidate value to render. If `v` is a string containing Jinja
- delimiters (``{{ ... }}``), it is rendered using the closure
- context ``ctx``; otherwise it is returned unchanged.
-
- Returns
- -------
- Any
- The rendered string when `v` is a templated string; otherwise the
- original value.
-
- Raises
- ------
- jinja2.UndefinedError
- If the template references an undefined variable and the Jinja
- environment uses ``StrictUndefined``.
- """
- if isinstance(v, str) and "{{" in v:
- return _JINJA.from_string(v).render(ctx)
- return v
-
- return {k: _render(v) for k, v in params.items()}
-
-
class JobConfig(BaseModelWithDot):
"""
Scheduler job options parsed from configuration.
@@ -698,16 +666,18 @@ class JobConfig(BaseModelWithDot):
model_config = ConfigDict()
- queue: str | None = None
- walltime: str | None = None
+ ntasks: int | None = None
nodes: int | None = Field(default=None, ge=1)
output_path: str | Path | None = None
+ queue: str | None = None
+ walltime: str | None = None
+ tasks: int | None = None
@field_validator("walltime")
@classmethod
def _hhmmss(cls, v: str | None) -> str | None:
"""
- Validate that ``walltime`` matches ``H:MM:SS`` or ``HH:MM:SS``.
+ Validate that ``walltime`` matches ``H:MM:SS`` or ``HH:MM:SS`` or ``HHH:MM:SS``.
Parameters
----------
@@ -755,9 +725,10 @@ class PismConfig(BaseModelWithDot):
Attributes
----------
- run : RunConfig
- Execution settings (launcher template, executable path/name,
- number of MPI ranks) with support for rendering Jinja placeholders.
+ campaign : CampaignConfig
+ Campaign-level metadata (data sources, forcing scenario, file references).
+ run_info : InfoConfig
+ Run metadata such as institution and title.
job : JobConfig
Scheduler options such as queue/partition, walltime, and number of
nodes. Unknown keys are forbidden in this section.
@@ -772,27 +743,37 @@ class PismConfig(BaseModelWithDot):
grid : GridConfig
Horizontal/vertical grid settings and registration. Derives
``grid.dx``/``grid.dy`` from ``resolution`` when not explicitly set.
- atmosphere : dict of str to Any, optional
- Additional atmosphere-related options to pass through (keys are
- typically dotted, e.g., ``"atmosphere.given.file"``). Defaults to ``{}``.
+ atmosphere : AtmosphereConfig
+ Atmosphere model selection and its option set.
+ ocean : OceanConfig
+ Ocean model selection and its option set.
+ surface : SurfaceConfig
+ Surface model selection and its option set.
+ frontal_melt : FrontalMeltConfig
+ Frontal melt model selection and its option set.
+ hydrology : HydrologyConfig
+ Hydrology model selection and its option set.
geometry : dict of str to Any, optional
Geometry-related options to pass through. Defaults to ``{}``.
- ocean : dict of str to Any, optional
- Ocean-related options to pass through. Defaults to ``{}``.
+ bed_deformation : BedDeformationConfig
+ Bed deformation model selection and its option set.
calving : dict of str to Any, optional
Calving-related options to pass through. Defaults to ``{}``.
iceflow : dict of str to Any, optional
Ice-flow-related options to pass through. Defaults to ``{}``.
- frontal_melt : dict of str to Any, optional
- Frontal melt-related options to pass through. Defaults to ``{}``.
- hydrology : dict of str to Any, optional
- Hydrology-related options to pass through. Defaults to ``{}``.
- surface : dict of str to Any, optional
- Surface-related options to pass through. Defaults to ``{}``.
reporting : dict of str to Any, optional
Reporting/output options to pass through. Defaults to ``{}``.
input : dict of str to Any, optional
Input file options to pass through. Defaults to ``{}``.
+ time_stepping : dict of str to Any, optional
+ Time-stepping-related options to pass through. Defaults to ``{}``.
+ inverse : dict of str to Any, optional
+ Inverse options to pass through. Defaults to ``{}``.
+ solver : dict of str to Any, optional
+ PETSc/Blatter solver knobs split by run kind, with ``forward`` and
+ ``inverse`` sub-tables (``[solver.forward]`` / ``[solver.inverse]``).
+ The ``forward`` half is injected into the forward ``pism`` command and
+ the ``inverse`` half into the ``pismi`` command. Defaults to ``{}``.
Notes
-----
@@ -813,24 +794,56 @@ class PismConfig(BaseModelWithDot):
"""
campaign: CampaignConfig
- run: RunConfig
run_info: InfoConfig
- job: JobConfig
+ job: JobConfig = Field(default_factory=JobConfig)
time: TimeConfig
energy: EnergyConfig
stress_balance: StressBalanceConfig
grid: GridConfig
atmosphere: AtmosphereConfig
+ ocean: OceanConfig
surface: SurfaceConfig
frontal_melt: FrontalMeltConfig
+ bed_deformation: BedDeformationConfig
hydrology: HydrologyConfig
geometry: dict[str, Any] = {}
- ocean: dict[str, Any] = {}
calving: dict[str, Any] = {}
iceflow: dict[str, Any] = {}
reporting: dict[str, Any] = {}
input: dict[str, Any] = {}
time_stepping: dict[str, Any] = {}
+ inverse: dict[str, Any] = {}
+ solver: dict[str, Any] = {}
+
+ @model_validator(mode="after")
+ def _expand_ismip7_counter(self) -> "PismConfig":
+ """
+ Expand ``run_info.counter`` into the derived experiment fields.
+
+ When an ISMIP7 Core Experiment counter (e.g. ``"C003"``) is set, it is the
+ single source of truth for the experiment identity: it fills
+ ``run_info.experiment``, ``campaign.pathway``, ``campaign.gcms``, and the
+ projection end (``time.end``) from
+ :data:`pism_terra.ismip7.experiments.CORE_EXPERIMENTS`. This runs for both
+ the staging and running entry points (both call :func:`load_config`), so a
+ single field drives the whole ISMIP7 pipeline. Non-counter (legacy) configs
+ are left untouched.
+
+ Returns
+ -------
+ PismConfig
+ The same instance, with counter-derived fields populated.
+ """
+ if not self.run_info.counter:
+ if self.time.time_end is None:
+ raise ValueError("time.end is required unless run_info.counter is set")
+ return self
+ spec = resolve_counter(self.run_info.counter)
+ self.run_info.experiment = spec.experiment_id
+ self.campaign.pathway = spec.pathway
+ self.campaign.gcms = [spec.esm_id]
+ self.time.time_end = f"{spec.proj_end_year}-01-01"
+ return self
class RestartConfig(BaseModelWithDot):
@@ -854,7 +867,7 @@ class InfoConfig(BaseModelWithDot):
"""
SECTION = "run_info"
- model_config = ConfigDict(populate_by_name=True, extra="ignore")
+ model_config = ConfigDict(populate_by_name=True)
institution: str = Field(
default="University of Alaska Fairbanks",
@@ -864,6 +877,22 @@ class InfoConfig(BaseModelWithDot):
default="PISM Campaign",
alias="run_info.title",
)
+ # Forwarded to PISM as run_info.* options (written as output global
+ # attributes, ISMIP7 section 5) AND used for submission naming (section 8).
+ group: str | None = Field(default=None, alias="run_info.group")
+ model: str | None = Field(default=None, alias="run_info.model")
+ contact_name: str | None = Field(default=None, alias="run_info.contact_name")
+ contact_email: str | None = Field(default=None, alias="run_info.contact_email")
+ # pism-terra-only ISMIP7 naming metadata (section 8); NOT PISM options, so
+ # they are kept out of the run command and consumed by the naming logic.
+ domain: str | None = Field(default=None, alias="run_info.domain")
+ set_id: str | None = Field(default=None, alias="run_info.set")
+ ism: str | None = Field(default=None, alias="run_info.ism")
+ experiment: str | None = Field(default=None, alias="run_info.experiment")
+ # ISMIP7 Core Experiment counter (e.g. "C003"). When set, PismConfig expands it
+ # into experiment/pathway/gcms/time.end (see PismConfig's counter resolver and
+ # pism_terra.ismip7.experiments). Naming-only, so kept out of _PISM_FIELDS.
+ counter: str | None = Field(default=None, alias="run_info.counter")
@staticmethod
def _quote(v: Any) -> str:
@@ -887,19 +916,38 @@ def _quote(v: Any) -> str:
s = s.replace("\\", "\\\\").replace('"', '\\"')
return f'"{s}"'
+ # Fields PISM recognizes as run_info.* options (written as output global
+ # attributes). The ISMIP7 naming-only fields (domain/set/ism/experiment)
+ # are intentionally excluded so they don't reach the PISM command.
+ _PISM_FIELDS: ClassVar[tuple[str, ...]] = (
+ "institution",
+ "title",
+ "group",
+ "model",
+ "contact_name",
+ "contact_email",
+ )
+
def as_params(self) -> dict[str, Any]:
"""
- Export run-info parameters with dotted aliases and quoted string values.
+ Export PISM run-info parameters with dotted aliases and quoted string values.
+
+ Emits only the fields PISM understands (``institution``, ``title``,
+ ``group``, ``model``, ``contact_name``, ``contact_email``), which it
+ writes as output global attributes. The ISMIP7 naming-only fields
+ (``domain``, ``set``, ``ism``, ``experiment``) are kept out of the PISM
+ command and consumed by the naming logic instead.
Returns
-------
dict[str, Any]
- Dictionary like ``{'run_info.institution': '\"Foo\"', 'run_info.title': '\"Bar\"'}``.
+ Dictionary like ``{'run_info.institution': '\"Foo\"', 'run_info.group': '\"UAF\"'}``.
"""
- out = self.model_dump(by_alias=True, exclude_none=True)
- for key in ("run_info.institution", "run_info.title"):
- if key in out and out[key] is not None:
- out[key] = self._quote(out[key])
+ out = {}
+ for name in self._PISM_FIELDS:
+ value = getattr(self, name)
+ if value is not None:
+ out[f"run_info.{name}"] = self._quote(value)
return out
@@ -919,6 +967,7 @@ class GridConfig(BaseModelWithDot):
Mz: int | None = Field(default=None, alias="grid.Mz")
extrapolation: str | None = Field(default=None, alias="grid.allow_extrapolation")
registration: str | None = Field(default=None, alias="grid.registration")
+ file: str | None = Field(default=None, alias="grid.file")
# derived / optionally provided:
dx: str | None = Field(default=None, alias="grid.dx")
@@ -1053,8 +1102,10 @@ class TimeConfig(BaseModelWithDot):
----------
time_start : str
Simulation start time (alias: ``"time.start"``).
- time_end : str
- Simulation end time (alias: ``"time.end"``).
+ time_end : str or None
+ Simulation end time (alias: ``"time.end"``). May be omitted when the run
+ is ISMIP7 counter-driven, in which case ``PismConfig`` fills it from the
+ Core Experiment's projection end year.
calendar : str or None
Calendar name (alias: ``"time.calendar"``), e.g., ``"standard"``.
reference_date : str or None
@@ -1065,7 +1116,7 @@ class TimeConfig(BaseModelWithDot):
model_config = ConfigDict(populate_by_name=True)
time_start: str = Field(alias="time.start")
- time_end: str = Field(alias="time.end")
+ time_end: str | None = Field(default=None, alias="time.end")
calendar: str | None = Field(default=None, alias="time.calendar")
reference_date: str | None = Field(default=None, alias="time.reference_date")
@@ -1134,6 +1185,16 @@ class AtmosphereConfig(ModelWithOptions):
SECTION = "atmosphere"
+class OceanConfig(ModelWithOptions):
+ """
+ Ocean model configuration.
+
+ Inherits fields/behavior from :class:`ModelWithOptions`.
+ """
+
+ SECTION = "ocean"
+
+
class SurfaceConfig(ModelWithOptions):
"""
Surface model configuration.
@@ -1164,6 +1225,16 @@ class HydrologyConfig(ModelWithOptions):
SECTION = "hydrology"
+class BedDeformationConfig(ModelWithOptions):
+ """
+ Bed deformation model configuration.
+
+ Inherits fields/behavior from :class:`ModelWithOptions`.
+ """
+
+ SECTION = "bed_deformation"
+
+
class FrontalMeltConfig(ModelWithOptions):
"""
Frontal melt model configuration.
@@ -1195,63 +1266,96 @@ class CampaignConfig(BaseModel):
Attributes
----------
- boot_file : str or None
- Path to the boot NetCDF file (relative to the input directory).
- outline_file : str or None
- Path to GPKG basin file (relative to the input directory).
+ bathymetry : str or None
+ bathymetry data source identifier (e.g., ``"gebco"``).
bucket : str or None
S3 bucket (e.g., ``"pism-cloud7-data"``).
climate : str or None
Climate forcing source identifier (e.g., ``"era5"``, ``"pmip4"``).
+ climatology : str or None
+ Climate forcing source identifier (e.g., ``"HIRHAM5-ERA5_YMM_1990_2019"``, ``"CARRA2_YMM"``).
dem : str or None
DEM data source identifier (e.g., ``"copernicus"``).
- end_year : str, float, or None
- End year of the forcing period.
+ forcing_mask : str or None
+ Forcing mask ("all", "glacier", "none").
velocity : str or None
Velocity data source identifier (e.g., ``"its_live"``).
- gcm : str, list, or None
+ heatflux : str or None
+ Heat-flow data source identifier (e.g., ``"lucazeau"``).
+ gcms : str, list, dict, or None
GCM model name(s) used for climate forcing.
boot_file : str or None
- Path to the grid NetCDF boot (relative to the input directory).
+ Path to the boot NetCDF file (relative to the input directory).
+ outline_file : str or None
+ Path to GPKG basin file (relative to the input directory).
grid_file : str or None
Path to the grid NetCDF file (relative to the input directory).
heatflux_file : str or None
- Path to the boot NetCDF file (relative to the input directory).
+ Path to the heat flux NetCDF file (relative to the input directory).
ice_thickness : str or None
Ice thickness data source identifier (e.g., ``"millan2022"``).
name : str or None
Human-readable campaign name.
+ ocean_file : str or None
+ Ocean forcing file name.
+ obs_file : str or None
+ Observations file name.
pathway : str or None
Forcing pathway or scenario identifier (e.g., ``"ssp585"``).
prefix : str or None
path to data in bucket (e.g., ``"ismip7_greenland_input"``).
+ present_day_forcings : str, list, or None
+ Present-day forcing identifier(s).
+ regrid_file : str or None
+ Path to a file used for regridding (relative to the input directory).
retreat_file : str or None
Path to the retreat NetCDF file (relative to the input directory).
- start_year : str, float, or None
- Start year of the forcing period.
+ rgi_complex_file : str or None
+ Filename of the RGI glacier-complex ("-C") outlines in the bucket.
+ rgi_glacier_file : str or None
+ Filename of the RGI glacier ("-G") outlines in the bucket.
+ historical_start_year : str, float, or None
+ First year of the historical forcing file.
+ historical_end_year : str, float, or None
+ Last (inclusive) year of the historical forcing file (e.g. 2014
+ under the ISMIP7 convention where projections start in 2015).
+ projection_start_year : str, float, or None
+ First year of the projection forcing file (e.g. 2015 for ISMIP7).
+ projection_end_year : str, float, or None
+ Last (inclusive) year of the projection forcing file. This value
+ differs per pathway (e.g. 2100 for ssp370, 2300 for ssp585).
version : str or None
Dataset or experiment version string.
"""
+ bathymetry: str | None = Field(default=None)
bucket: str | None = Field(default=None)
climate: str | None = Field(default=None)
+ climatology: str | None = Field(default=None)
dem: str | None = Field(default=None)
+ forcing_mask: str | None = Field(default=None)
velocity: str | None = Field(default=None)
- gcms: str | list | None = Field(default=None)
- present_day_forcings: str | list | None = Field(default=None)
- future_forcings: str | list | None = Field(default=None)
+ heatflux: str | None = Field(default=None)
+ gcms: str | list | dict | None = Field(default=None)
boot_file: str | None = Field(default=None)
outline_file: str | None = Field(default=None)
grid_file: str | None = Field(default=None)
heatflux_file: str | None = Field(default=None)
ice_thickness: str | None = Field(default=None)
name: str | None = Field(default=None)
+ obs_file: str | None = Field(default=None)
+ ocean_file: str | None = Field(default=None)
pathway: str | None = Field(default=None)
prefix: str | None = Field(default=None)
+ present_day_forcings: str | list | None = Field(default=None)
regrid_file: str | None = Field(default=None)
retreat_file: str | None = Field(default=None)
- rgi_file: str | None = Field(default=None)
- start_year: str | float | None = Field(default=None)
+ rgi_complex_file: str | None = Field(default=None)
+ rgi_glacier_file: str | None = Field(default=None)
+ historical_start_year: str | float | None = Field(default=None)
+ historical_end_year: str | float | None = Field(default=None)
+ projection_start_year: str | float | None = Field(default=None)
+ projection_end_year: str | float | None = Field(default=None)
version: str | None = Field(default=None)
def as_params(self, **extra: Any) -> dict[str, Any]:
@@ -1293,12 +1397,6 @@ def _render(v: Any) -> Any:
Any
The rendered string when `v` is a templated string; otherwise the
original value.
-
- Raises
- ------
- jinja2.UndefinedError
- If the template references an undefined variable and the Jinja
- environment uses ``StrictUndefined``.
"""
if isinstance(v, str) and "{{" in v:
return _JINJA.from_string(v).render(ctx)
diff --git a/pism_terra/config/S4F_target_AK_RGI_id.csv b/pism_terra/config/S4F_target_AK_RGI_id.csv
new file mode 100644
index 0000000..b6c3a99
--- /dev/null
+++ b/pism_terra/config/S4F_target_AK_RGI_id.csv
@@ -0,0 +1,166 @@
+rgi_id
+RGI2000-v7.0-G-01-02402
+RGI2000-v7.0-G-01-02411
+RGI2000-v7.0-G-01-02509
+RGI2000-v7.0-G-01-02580
+RGI2000-v7.0-G-01-02593
+RGI2000-v7.0-G-01-03908
+RGI2000-v7.0-G-01-03926
+RGI2000-v7.0-G-01-03935
+RGI2000-v7.0-G-01-03986
+RGI2000-v7.0-G-01-04015
+RGI2000-v7.0-G-01-04027
+RGI2000-v7.0-G-01-04282
+RGI2000-v7.0-G-01-04294
+RGI2000-v7.0-G-01-04310
+RGI2000-v7.0-G-01-04360
+RGI2000-v7.0-G-01-04433
+RGI2000-v7.0-G-01-04458
+RGI2000-v7.0-G-01-05669
+RGI2000-v7.0-G-01-05702
+RGI2000-v7.0-G-01-05703
+RGI2000-v7.0-G-01-05740
+RGI2000-v7.0-G-01-05749
+RGI2000-v7.0-G-01-05780
+RGI2000-v7.0-G-01-05878
+RGI2000-v7.0-G-01-05906
+RGI2000-v7.0-G-01-05966
+RGI2000-v7.0-G-01-06010
+RGI2000-v7.0-G-01-06044
+RGI2000-v7.0-G-01-06051
+RGI2000-v7.0-G-01-06058
+RGI2000-v7.0-G-01-06333
+RGI2000-v7.0-G-01-06338
+RGI2000-v7.0-G-01-06340
+RGI2000-v7.0-G-01-07671
+RGI2000-v7.0-G-01-07676
+RGI2000-v7.0-G-01-07715
+RGI2000-v7.0-G-01-07898
+RGI2000-v7.0-G-01-07942
+RGI2000-v7.0-G-01-08014
+RGI2000-v7.0-G-01-08602
+RGI2000-v7.0-G-01-08603
+RGI2000-v7.0-G-01-08604
+RGI2000-v7.0-G-01-08628
+RGI2000-v7.0-G-01-08669
+RGI2000-v7.0-G-01-08757
+RGI2000-v7.0-G-01-08822
+RGI2000-v7.0-G-01-08848
+RGI2000-v7.0-G-01-08928
+RGI2000-v7.0-G-01-09260
+RGI2000-v7.0-G-01-09262
+RGI2000-v7.0-G-01-09288
+RGI2000-v7.0-G-01-09340
+RGI2000-v7.0-G-01-09463
+RGI2000-v7.0-G-01-09511
+RGI2000-v7.0-G-01-09669
+RGI2000-v7.0-G-01-10024
+RGI2000-v7.0-G-01-10218
+RGI2000-v7.0-G-01-10395
+RGI2000-v7.0-G-01-10398
+RGI2000-v7.0-G-01-10406
+RGI2000-v7.0-G-01-10713
+RGI2000-v7.0-G-01-10744
+RGI2000-v7.0-G-01-11568
+RGI2000-v7.0-G-01-11577
+RGI2000-v7.0-G-01-11603
+RGI2000-v7.0-G-01-11622
+RGI2000-v7.0-G-01-11628
+RGI2000-v7.0-G-01-11711
+RGI2000-v7.0-G-01-11722
+RGI2000-v7.0-G-01-12420
+RGI2000-v7.0-G-01-12440
+RGI2000-v7.0-G-01-12441
+RGI2000-v7.0-G-01-12443
+RGI2000-v7.0-G-01-12946
+RGI2000-v7.0-G-01-12954
+RGI2000-v7.0-G-01-12965
+RGI2000-v7.0-G-01-13099
+RGI2000-v7.0-G-01-13110
+RGI2000-v7.0-G-01-13326
+RGI2000-v7.0-G-01-13345
+RGI2000-v7.0-G-01-13350
+RGI2000-v7.0-G-01-13420
+RGI2000-v7.0-G-01-13555
+RGI2000-v7.0-G-01-13600
+RGI2000-v7.0-G-01-13666
+RGI2000-v7.0-G-01-13838
+RGI2000-v7.0-G-01-13930
+RGI2000-v7.0-G-01-14180
+RGI2000-v7.0-G-01-14439
+RGI2000-v7.0-G-01-14521
+RGI2000-v7.0-G-01-14716
+RGI2000-v7.0-G-01-14761
+RGI2000-v7.0-G-01-14767
+RGI2000-v7.0-G-01-14844
+RGI2000-v7.0-G-01-14859
+RGI2000-v7.0-G-01-14910
+RGI2000-v7.0-G-01-14965
+RGI2000-v7.0-G-01-15025
+RGI2000-v7.0-G-01-15127
+RGI2000-v7.0-G-01-15252
+RGI2000-v7.0-G-01-15293
+RGI2000-v7.0-G-01-15351
+RGI2000-v7.0-G-01-15382
+RGI2000-v7.0-G-01-15524
+RGI2000-v7.0-G-01-15552
+RGI2000-v7.0-G-01-15555
+RGI2000-v7.0-G-01-16227
+RGI2000-v7.0-G-01-16236
+RGI2000-v7.0-G-01-16307
+RGI2000-v7.0-G-01-16413
+RGI2000-v7.0-G-01-16437
+RGI2000-v7.0-G-01-16759
+RGI2000-v7.0-G-01-16822
+RGI2000-v7.0-G-01-16842
+RGI2000-v7.0-G-01-16905
+RGI2000-v7.0-G-01-16911
+RGI2000-v7.0-G-01-16980
+RGI2000-v7.0-G-01-16987
+RGI2000-v7.0-G-01-17002
+RGI2000-v7.0-G-01-17004
+RGI2000-v7.0-G-01-17086
+RGI2000-v7.0-G-01-17115
+RGI2000-v7.0-G-01-18869
+RGI2000-v7.0-G-01-19246
+RGI2000-v7.0-G-01-19277
+RGI2000-v7.0-G-01-19287
+RGI2000-v7.0-G-01-19370
+RGI2000-v7.0-G-01-19371
+RGI2000-v7.0-G-01-19394
+RGI2000-v7.0-G-01-19425
+RGI2000-v7.0-G-01-19432
+RGI2000-v7.0-G-01-19647
+RGI2000-v7.0-G-01-19709
+RGI2000-v7.0-G-01-19712
+RGI2000-v7.0-G-01-19713
+RGI2000-v7.0-G-01-19843
+RGI2000-v7.0-G-01-20403
+RGI2000-v7.0-G-01-20404
+RGI2000-v7.0-G-01-20441
+RGI2000-v7.0-G-01-20702
+RGI2000-v7.0-G-01-20851
+RGI2000-v7.0-G-01-21229
+RGI2000-v7.0-G-01-21357
+RGI2000-v7.0-G-01-21451
+RGI2000-v7.0-G-01-21469
+RGI2000-v7.0-G-01-21496
+RGI2000-v7.0-G-01-21534
+RGI2000-v7.0-G-01-21559
+RGI2000-v7.0-G-01-25753
+RGI2000-v7.0-G-01-25758
+RGI2000-v7.0-G-01-26047
+RGI2000-v7.0-G-01-26469
+RGI2000-v7.0-G-01-26477
+RGI2000-v7.0-G-01-26478
+RGI2000-v7.0-G-01-26886
+RGI2000-v7.0-G-01-26998
+RGI2000-v7.0-G-01-27011
+RGI2000-v7.0-G-01-27051
+RGI2000-v7.0-G-01-27071
+RGI2000-v7.0-G-01-27133
+RGI2000-v7.0-G-01-27166
+RGI2000-v7.0-G-01-27210
+RGI2000-v7.0-G-01-27252
+RGI2000-v7.0-G-01-27355
+RGI2000-v7.0-G-01-27357
diff --git a/pism_terra/config/S4F_target_CA_RGI_id.csv b/pism_terra/config/S4F_target_CA_RGI_id.csv
new file mode 100644
index 0000000..d3bcbd7
--- /dev/null
+++ b/pism_terra/config/S4F_target_CA_RGI_id.csv
@@ -0,0 +1,220 @@
+rgi_id
+RGI2000-v7.0-G-03-00160
+RGI2000-v7.0-G-03-00165
+RGI2000-v7.0-G-03-00183
+RGI2000-v7.0-G-03-00191
+RGI2000-v7.0-G-03-00197
+RGI2000-v7.0-G-03-00199
+RGI2000-v7.0-G-03-00200
+RGI2000-v7.0-G-03-00256
+RGI2000-v7.0-G-03-00270
+RGI2000-v7.0-G-03-00273
+RGI2000-v7.0-G-03-00344
+RGI2000-v7.0-G-03-00356
+RGI2000-v7.0-G-03-00380
+RGI2000-v7.0-G-03-00389
+RGI2000-v7.0-G-03-00437
+RGI2000-v7.0-G-03-00439
+RGI2000-v7.0-G-03-00462
+RGI2000-v7.0-G-03-00479
+RGI2000-v7.0-G-03-00497
+RGI2000-v7.0-G-03-00500
+RGI2000-v7.0-G-03-00501
+RGI2000-v7.0-G-03-00502
+RGI2000-v7.0-G-03-00503
+RGI2000-v7.0-G-03-00512
+RGI2000-v7.0-G-03-00513
+RGI2000-v7.0-G-03-00517
+RGI2000-v7.0-G-03-00591
+RGI2000-v7.0-G-03-00592
+RGI2000-v7.0-G-03-00670
+RGI2000-v7.0-G-03-00885
+RGI2000-v7.0-G-03-00931
+RGI2000-v7.0-G-03-00942
+RGI2000-v7.0-G-03-00961
+RGI2000-v7.0-G-03-00977
+RGI2000-v7.0-G-03-00980
+RGI2000-v7.0-G-03-00983
+RGI2000-v7.0-G-03-01014
+RGI2000-v7.0-G-03-01198
+RGI2000-v7.0-G-03-01221
+RGI2000-v7.0-G-03-01224
+RGI2000-v7.0-G-03-01236
+RGI2000-v7.0-G-03-01239
+RGI2000-v7.0-G-03-01288
+RGI2000-v7.0-G-03-01358
+RGI2000-v7.0-G-03-01364
+RGI2000-v7.0-G-03-01389
+RGI2000-v7.0-G-03-01440
+RGI2000-v7.0-G-03-01481
+RGI2000-v7.0-G-03-01558
+RGI2000-v7.0-G-03-01574
+RGI2000-v7.0-G-03-01575
+RGI2000-v7.0-G-03-01596
+RGI2000-v7.0-G-03-01629
+RGI2000-v7.0-G-03-01678
+RGI2000-v7.0-G-03-01680
+RGI2000-v7.0-G-03-01682
+RGI2000-v7.0-G-03-01749
+RGI2000-v7.0-G-03-01753
+RGI2000-v7.0-G-03-01754
+RGI2000-v7.0-G-03-01774
+RGI2000-v7.0-G-03-01811
+RGI2000-v7.0-G-03-01812
+RGI2000-v7.0-G-03-01814
+RGI2000-v7.0-G-03-01815
+RGI2000-v7.0-G-03-01816
+RGI2000-v7.0-G-03-01830
+RGI2000-v7.0-G-03-01853
+RGI2000-v7.0-G-03-02174
+RGI2000-v7.0-G-03-02186
+RGI2000-v7.0-G-03-02190
+RGI2000-v7.0-G-03-02191
+RGI2000-v7.0-G-03-02239
+RGI2000-v7.0-G-03-02242
+RGI2000-v7.0-G-03-02282
+RGI2000-v7.0-G-03-02283
+RGI2000-v7.0-G-03-02310
+RGI2000-v7.0-G-03-02335
+RGI2000-v7.0-G-03-02336
+RGI2000-v7.0-G-03-02385
+RGI2000-v7.0-G-03-02446
+RGI2000-v7.0-G-03-02481
+RGI2000-v7.0-G-03-02505
+RGI2000-v7.0-G-03-02526
+RGI2000-v7.0-G-03-02528
+RGI2000-v7.0-G-03-02540
+RGI2000-v7.0-G-03-02561
+RGI2000-v7.0-G-03-02654
+RGI2000-v7.0-G-03-02671
+RGI2000-v7.0-G-03-02672
+RGI2000-v7.0-G-03-02673
+RGI2000-v7.0-G-03-02675
+RGI2000-v7.0-G-03-02798
+RGI2000-v7.0-G-03-02802
+RGI2000-v7.0-G-03-02803
+RGI2000-v7.0-G-03-02813
+RGI2000-v7.0-G-03-02818
+RGI2000-v7.0-G-03-02819
+RGI2000-v7.0-G-03-02822
+RGI2000-v7.0-G-03-02825
+RGI2000-v7.0-G-03-02829
+RGI2000-v7.0-G-03-02848
+RGI2000-v7.0-G-03-02849
+RGI2000-v7.0-G-03-02887
+RGI2000-v7.0-G-03-02908
+RGI2000-v7.0-G-03-02909
+RGI2000-v7.0-G-03-02930
+RGI2000-v7.0-G-03-02931
+RGI2000-v7.0-G-03-02935
+RGI2000-v7.0-G-03-02940
+RGI2000-v7.0-G-03-02941
+RGI2000-v7.0-G-03-02944
+RGI2000-v7.0-G-03-02945
+RGI2000-v7.0-G-03-02965
+RGI2000-v7.0-G-03-02967
+RGI2000-v7.0-G-03-03003
+RGI2000-v7.0-G-03-03014
+RGI2000-v7.0-G-03-03016
+RGI2000-v7.0-G-03-03017
+RGI2000-v7.0-G-03-03018
+RGI2000-v7.0-G-03-03041
+RGI2000-v7.0-G-03-03045
+RGI2000-v7.0-G-03-03051
+RGI2000-v7.0-G-03-03087
+RGI2000-v7.0-G-03-03095
+RGI2000-v7.0-G-03-03124
+RGI2000-v7.0-G-03-03126
+RGI2000-v7.0-G-03-03128
+RGI2000-v7.0-G-03-03186
+RGI2000-v7.0-G-03-04173
+RGI2000-v7.0-G-03-04174
+RGI2000-v7.0-G-03-04194
+RGI2000-v7.0-G-03-04202
+RGI2000-v7.0-G-03-04203
+RGI2000-v7.0-G-03-04204
+RGI2000-v7.0-G-03-04205
+RGI2000-v7.0-G-03-04206
+RGI2000-v7.0-G-03-04207
+RGI2000-v7.0-G-03-04208
+RGI2000-v7.0-G-03-04209
+RGI2000-v7.0-G-03-04210
+RGI2000-v7.0-G-03-04211
+RGI2000-v7.0-G-03-04212
+RGI2000-v7.0-G-03-04217
+RGI2000-v7.0-G-03-04218
+RGI2000-v7.0-G-03-04220
+RGI2000-v7.0-G-03-04225
+RGI2000-v7.0-G-03-04257
+RGI2000-v7.0-G-03-04258
+RGI2000-v7.0-G-03-04260
+RGI2000-v7.0-G-03-04270
+RGI2000-v7.0-G-03-04271
+RGI2000-v7.0-G-03-04275
+RGI2000-v7.0-G-03-04276
+RGI2000-v7.0-G-03-04280
+RGI2000-v7.0-G-03-04281
+RGI2000-v7.0-G-03-04289
+RGI2000-v7.0-G-03-04290
+RGI2000-v7.0-G-03-04291
+RGI2000-v7.0-G-03-04292
+RGI2000-v7.0-G-03-04305
+RGI2000-v7.0-G-03-04311
+RGI2000-v7.0-G-03-04312
+RGI2000-v7.0-G-03-04313
+RGI2000-v7.0-G-03-04314
+RGI2000-v7.0-G-03-04315
+RGI2000-v7.0-G-03-04323
+RGI2000-v7.0-G-03-04324
+RGI2000-v7.0-G-03-04325
+RGI2000-v7.0-G-03-04326
+RGI2000-v7.0-G-03-04327
+RGI2000-v7.0-G-03-04329
+RGI2000-v7.0-G-03-04339
+RGI2000-v7.0-G-03-04340
+RGI2000-v7.0-G-03-04341
+RGI2000-v7.0-G-03-04342
+RGI2000-v7.0-G-03-04354
+RGI2000-v7.0-G-03-04357
+RGI2000-v7.0-G-03-04358
+RGI2000-v7.0-G-03-04800
+RGI2000-v7.0-G-03-04803
+RGI2000-v7.0-G-03-04804
+RGI2000-v7.0-G-03-04808
+RGI2000-v7.0-G-03-04809
+RGI2000-v7.0-G-03-04816
+RGI2000-v7.0-G-03-04820
+RGI2000-v7.0-G-03-04837
+RGI2000-v7.0-G-03-04852
+RGI2000-v7.0-G-03-04883
+RGI2000-v7.0-G-03-04903
+RGI2000-v7.0-G-03-04910
+RGI2000-v7.0-G-03-04912
+RGI2000-v7.0-G-03-04924
+RGI2000-v7.0-G-03-04927
+RGI2000-v7.0-G-03-04928
+RGI2000-v7.0-G-03-04931
+RGI2000-v7.0-G-03-05053
+RGI2000-v7.0-G-03-05101
+RGI2000-v7.0-G-03-05107
+RGI2000-v7.0-G-03-05127
+RGI2000-v7.0-G-03-05128
+RGI2000-v7.0-G-03-05138
+RGI2000-v7.0-G-03-05150
+RGI2000-v7.0-G-03-05155
+RGI2000-v7.0-G-03-05156
+RGI2000-v7.0-G-03-05168
+RGI2000-v7.0-G-03-05169
+RGI2000-v7.0-G-03-05172
+RGI2000-v7.0-G-03-05173
+RGI2000-v7.0-G-03-05174
+RGI2000-v7.0-G-03-05190
+RGI2000-v7.0-G-03-05191
+RGI2000-v7.0-G-03-05192
+RGI2000-v7.0-G-03-05193
+RGI2000-v7.0-G-03-05199
+RGI2000-v7.0-G-03-05203
+RGI2000-v7.0-G-03-05204
+RGI2000-v7.0-G-03-05205
+RGI2000-v7.0-G-03-05206
+RGI2000-v7.0-G-03-05208
diff --git a/pism_terra/config/S4F_target_SV_RGI_id.csv b/pism_terra/config/S4F_target_SV_RGI_id.csv
new file mode 100644
index 0000000..6639fc2
--- /dev/null
+++ b/pism_terra/config/S4F_target_SV_RGI_id.csv
@@ -0,0 +1,121 @@
+rgi_id
+RGI2000-v7.0-G-07-00010
+RGI2000-v7.0-G-07-00054
+RGI2000-v7.0-G-07-00082
+RGI2000-v7.0-G-07-00084
+RGI2000-v7.0-G-07-00241
+RGI2000-v7.0-G-07-00243
+RGI2000-v7.0-G-07-00245
+RGI2000-v7.0-G-07-00246
+RGI2000-v7.0-G-07-00249
+RGI2000-v7.0-G-07-00250
+RGI2000-v7.0-G-07-00251
+RGI2000-v7.0-G-07-00252
+RGI2000-v7.0-G-07-00253
+RGI2000-v7.0-G-07-00254
+RGI2000-v7.0-G-07-00256
+RGI2000-v7.0-G-07-00257
+RGI2000-v7.0-G-07-00281
+RGI2000-v7.0-G-07-00291
+RGI2000-v7.0-G-07-00307
+RGI2000-v7.0-G-07-00308
+RGI2000-v7.0-G-07-00309
+RGI2000-v7.0-G-07-00354
+RGI2000-v7.0-G-07-00355
+RGI2000-v7.0-G-07-00391
+RGI2000-v7.0-G-07-00774
+RGI2000-v7.0-G-07-00776
+RGI2000-v7.0-G-07-00807
+RGI2000-v7.0-G-07-00808
+RGI2000-v7.0-G-07-00819
+RGI2000-v7.0-G-07-00831
+RGI2000-v7.0-G-07-00876
+RGI2000-v7.0-G-07-00889
+RGI2000-v7.0-G-07-00899
+RGI2000-v7.0-G-07-00902
+RGI2000-v7.0-G-07-00913
+RGI2000-v7.0-G-07-01070
+RGI2000-v7.0-G-07-01071
+RGI2000-v7.0-G-07-01072
+RGI2000-v7.0-G-07-01073
+RGI2000-v7.0-G-07-01074
+RGI2000-v7.0-G-07-01075
+RGI2000-v7.0-G-07-01077
+RGI2000-v7.0-G-07-01082
+RGI2000-v7.0-G-07-01097
+RGI2000-v7.0-G-07-01101
+RGI2000-v7.0-G-07-01102
+RGI2000-v7.0-G-07-01103
+RGI2000-v7.0-G-07-01104
+RGI2000-v7.0-G-07-01109
+RGI2000-v7.0-G-07-01117
+RGI2000-v7.0-G-07-01119
+RGI2000-v7.0-G-07-01129
+RGI2000-v7.0-G-07-01132
+RGI2000-v7.0-G-07-01147
+RGI2000-v7.0-G-07-01149
+RGI2000-v7.0-G-07-01160
+RGI2000-v7.0-G-07-01161
+RGI2000-v7.0-G-07-01163
+RGI2000-v7.0-G-07-01167
+RGI2000-v7.0-G-07-01169
+RGI2000-v7.0-G-07-01211
+RGI2000-v7.0-G-07-01212
+RGI2000-v7.0-G-07-01219
+RGI2000-v7.0-G-07-01221
+RGI2000-v7.0-G-07-01222
+RGI2000-v7.0-G-07-01268
+RGI2000-v7.0-G-07-01269
+RGI2000-v7.0-G-07-01312
+RGI2000-v7.0-G-07-01316
+RGI2000-v7.0-G-07-01323
+RGI2000-v7.0-G-07-01324
+RGI2000-v7.0-G-07-01332
+RGI2000-v7.0-G-07-01333
+RGI2000-v7.0-G-07-01334
+RGI2000-v7.0-G-07-01335
+RGI2000-v7.0-G-07-01337
+RGI2000-v7.0-G-07-01338
+RGI2000-v7.0-G-07-01339
+RGI2000-v7.0-G-07-01340
+RGI2000-v7.0-G-07-01342
+RGI2000-v7.0-G-07-01343
+RGI2000-v7.0-G-07-01344
+RGI2000-v7.0-G-07-01363
+RGI2000-v7.0-G-07-01364
+RGI2000-v7.0-G-07-01365
+RGI2000-v7.0-G-07-01366
+RGI2000-v7.0-G-07-01367
+RGI2000-v7.0-G-07-01368
+RGI2000-v7.0-G-07-01369
+RGI2000-v7.0-G-07-01370
+RGI2000-v7.0-G-07-01371
+RGI2000-v7.0-G-07-01372
+RGI2000-v7.0-G-07-01373
+RGI2000-v7.0-G-07-01374
+RGI2000-v7.0-G-07-01375
+RGI2000-v7.0-G-07-01376
+RGI2000-v7.0-G-07-01378
+RGI2000-v7.0-G-07-01379
+RGI2000-v7.0-G-07-01380
+RGI2000-v7.0-G-07-01381
+RGI2000-v7.0-G-07-01382
+RGI2000-v7.0-G-07-01383
+RGI2000-v7.0-G-07-01385
+RGI2000-v7.0-G-07-01386
+RGI2000-v7.0-G-07-01387
+RGI2000-v7.0-G-07-01401
+RGI2000-v7.0-G-07-01405
+RGI2000-v7.0-G-07-01407
+RGI2000-v7.0-G-07-01408
+RGI2000-v7.0-G-07-01459
+RGI2000-v7.0-G-07-01466
+RGI2000-v7.0-G-07-01467
+RGI2000-v7.0-G-07-01531
+RGI2000-v7.0-G-07-01540
+RGI2000-v7.0-G-07-01541
+RGI2000-v7.0-G-07-01542
+RGI2000-v7.0-G-07-01544
+RGI2000-v7.0-G-07-01545
+RGI2000-v7.0-G-07-01546
+RGI2000-v7.0-G-07-01548
diff --git a/pism_terra/config/bm_greenland_historical.toml b/pism_terra/config/bm_greenland_historical.toml
deleted file mode 100644
index 04e1c0d..0000000
--- a/pism_terra/config/bm_greenland_historical.toml
+++ /dev/null
@@ -1,203 +0,0 @@
-[run]
-
-executable = "pism"
-mpi = "mpirun -np {{ ntasks }}"
-ntasks = 80
-writer = ""
-
-[job]
-
-queue = "t2small"
-walltime = "48:00:00"
-nodes = 1
-
-[run_info]
-
-'run_info.institution' = "University of Alaska Fairbanks"
-'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
-
-[campaign]
-
-bucket = "pism-cloud-data"
-prefix = "bedmachine_greenland_input"
-pathway = "historical"
-version = "1"
-start_year = "1975"
-end_year = "2015"
-gcm = "CESM2-WACCM"
-boot_file = "boot_g600m_GreenlandObsISMIP7-v1.3.nc"
-grid_file = "ismip7_greenland_grid.nc"
-regrid_file = "g600m_id_BAYES-MEDIAN_1980-1-1_1984-12-31.nc"
-retreat_file = "pism_g600m_frontretreat_calfin_1972_2019_ME.nc"
-heatflux_file = "heatflux_g600m_GreenlandObsISMIP7-v1.3.nc"
-
-[time]
-
-'time.start' = "1980-01-01"
-'time.end' = "2015-01-01"
-'time.calendar' = "standard"
-'time.reference_date' = "1978-01-01"
-
-[time_stepping]
-
-'time_stepping.adaptive_ratio' = 0.12
-'time_stepping.skip.enabled' = "yes"
-'time_stepping.skip.max' = 100
-
-['geometry']
-
-'geometry.front_retreat.use_cfl' = 'yes'
-'geometry.part_grid.enabled' = 'yes'
-'geometry.remove_icebergs' = 'yes'
-'geometry.front_retreat.prescribed.file' = "none"
-
-['input']
-
-'input.bootstrap' = "yes"
-'input.forcing.time_extrapolation' = "yes"
-'input.forcing.buffer_size' = 37
-'input.regrid.file' = "none"
-'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume"
-
-[reporting]
-
-'output.format' = "netcdf4_serial"
-'output.compression_level' = 2
-'output.size' = "medium"
-'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
-'output.scalar.file' = "none"
-'output.scalar.times' = 'daily'
-'output.spatial.file' = "none"
-'output.spatial.times' = "monthly"
-'output.spatial.vars' = "bmelt,effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,tauc,tillwat,diffusivity"
-'output.checkpoint.interval' = 24
-'output.extra.stop_missing' = 'no'
-'output.ISMIP6' = "no"
-
-[grid]
-
-resolution = "900m"
-'grid.Lbz' = 2000
-'grid.Lz' = 4000
-'grid.Mbz' = 51
-'grid.Mz' = 201
-'grid.registration' = "center"
-'grid.allow_extrapolation' = "yes"
-
-[energy]
-
-model = "enthalpy"
-
-[energy.options.enthalpy]
-
-'energy.model' = "enthalpy"
-'energy.bedrock_thermal.file' = "none"
-'stress_balance.blatter.flow_law' = "gpbld"
-'stress_balance.sia.flow_law' = "gpbld"
-'stress_balance.ssa.flow_law' = "gpbld"
-
-[stress_balance]
-
-model = 'hybrid'
-
-[stress_balance.options.blatter]
-
-'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.max_diffusivity' = 1000000.0
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
-
-[stress_balance.options.hybrid]
-
-'stress_balance.model' = "ssa+sia"
-'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.max_diffusivity' = 100000.0
-
-[iceflow]
-
-'basal_resistance.pseudo_plastic.enabled' = 'yes'
-'basal_resistance.pseudo_plastic.q' = 0.7508221
-'basal_yield_stress.model' = 'mohr_coulomb'
-'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
-'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
-'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
-'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
-'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
-'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
-'stress_balance.blatter.enhancement_factor' = 2.608046
-'stress_balance.sia.enhancement_factor' = 2.608046
-'stress_balance.ssa.Glen_exponent' = 3.309718
-
-[surface]
-
-model = "given"
-
-[surface.options.given]
-
-'surface.models' = "given"
-'surface.given.file' = "none"
-
-[calving]
-
-'calving.methods' = 'vonmises_calving'
-'calving.vonmises_calving.sigma_max' = 7.5e5
-
-[atmosphere]
-
-model = "given"
-
-[atmosphere.options.given]
-
-'atmosphere.models' = "given"
-'atmosphere.given.file' = "none"
-
-[ocean]
-
-'ocean.models' = "th"
-'ocean.th.file' = "none"
-'ocean.th.clip_salinity' = "False"
-
-[hydrology]
-
-model = "routing"
-
-[hydrology.options.null]
-
-'hydrology.model' = "null"
-'hydrology.null_diffuse_till_water' = "yes"
-
-[hydrology.options.routing]
-
-'hydrology.model' = "routing"
-'hydrology.routing.include_floating_ice' = "yes"
-'hydrology.surface_input.file' = "none"
-
-
-[frontal_melt]
-
-model = "off"
-
-[frontal_melt.options.off]
-
-[frontal_melt.options.routing]
-
-'frontal_melt.routing.file' = "none"
-'frontal_melt.routing.parameter_a' = 3e-4
-'frontal_melt.routing.parameter_b' = 0.15
-'frontal_melt.routing.power_alpha' = 0.39
-'frontal_melt.routing.power_beta' = 1.18
diff --git a/pism_terra/config/cosipy_m2.toml b/pism_terra/config/cosipy_m2.toml
deleted file mode 100644
index d5efd9d..0000000
--- a/pism_terra/config/cosipy_m2.toml
+++ /dev/null
@@ -1,112 +0,0 @@
-[run]
-
-exec = "/Users/andy/pism-dev-conda-petsc/bin/pism"
-mpi = "mpirun -np"
-ntasks = 8
-queue = "t2small"
-walltime = "12:00:00"
-
-[campaign]
-
-setup = "init"
-
-[climate]
-
-dataset="reanalysis-era5-land-monthly-means"
-
-[grid]
-
-resolution = "200m"
-'grid.Lbz' = 0
-'grid.Lz' = 2000
-'grid.Mbz' = 1
-'grid.Mz' = 101
-'grid.registration' = "center"
-
-['geometry']
-
-'geometry.front_retreat.use_cfl' = 'yes'
-'geometry.part_grid.enabled' = 'yes'
-'geometry.remove_icebergs' = 'yes'
-
-['input']
-
-'input.bootstrap' = "yes"
-'input.forcing.buffer_size' = 400
-
-[time]
-
-'time.start' = "1981-10-01"
-'time.end' = "2001-01-01"
-'time.calendar' = "standard"
-'time.reference_date' = "1980-01-01"
-'time_stepping.adaptive_ratio' = 250
-'time_stepping.skip.enabled' = "yes"
-'time_stepping.skip.max' = 100
-
-[reporting]
-
-'output.format' = "netcdf4_parallel"
-'output.size' = "medium"
-'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf,bmelt"
-'output.scalar.times' = "daily"
-'output.spatial.file' = "none"
-'output.spatial.times' = "monthly"
-'output.spatial.vars' = "climatic_mass_balance,thk,velsurf_mag,velbase_mag,ice_mass,mass_fluxes,mask,usurf"
-'output.checkpoint.interval' = 24
-
-[energy]
-
-model = "enthalpy"
-
-[energy.options]
-
-none = { 'energy.model' = "none", 'flow_law.isothermal_Glen.ice_softness' = 1.0e-24, 'stress_balance.blatter.flow_law' = "isothermal_glen", 'stress_balance.sia.flow_law' = "isothermal_glen", 'stress_balance.ssa.flow_law' = "isothermal_glen"}
-enthalpy = { 'energy.model' = "enthalpy", 'flow_law.isothermal_Glen.ice_softness' = 1.0e-24, 'stress_balance.blatter.flow_law' = "gpbld", 'stress_balance.sia.flow_law' = "gpbld", 'stress_balance.ssa.flow_law' = "gpbld"}
-
-[stress_balance]
-
-'stress_balance.model' = 'blatter'
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
-'stress_balance.blatter.use_eta_transform' = 'yes'
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.surface_gradient_method' = 'eta'
-'stress_balance.sia.max_diffusivity' = 100000.0
-
-[stress_balance.options]
-
-blatter = {'stress_balance.model' = "blatter", bp_ksp_monitor = "", bp_ksp_view_singularvalues = "", bp_snes_monitor_ratio = "", bp_pc_type = "mg", bp_mg_levels_ksp_type = "richardson", bp_mg_levels_pc_type ="sor", bp_mg_coarse_ksp_type = "preonly", bp_mg_coarse_pc_type = "lu", bp_pc_mg_levels = 3, bp_pc_type mg = "", bp_snes_ksp_ew = 1, bp_snes_ksp_ew_version = 3}
-
-hybrid = {'stress_balance.model' = "ssa+sia", 'stress_balance.ssa.method' = "fd"}
-
-[iceflow]
-
-'basal_resistance.pseudo_plastic.q' = 0.75
-'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
-'basal_resistance.pseudo_plastic.enabled' = 'yes'
-'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
-'basal_yield_stress.model' = 'mohr_coulomb'
-'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
-'stress_balance.blatter.enhancement_factor' = 2.0
-'stress_balance.sia.enhancement_factor' = 2.0
-
-['surface']
-
-'surface.models' = 'given'
-'surface.given.file' = "none"
-
-['calving']
-
-'calving.methods' = 'float_kill'
-
-['atmosphere']
-
-'atmosphere.models' = "given"
-'atmosphere.given.file' = "none"
-'atmosphere.elevation_change.temperature_lapse_rate' = 6
-
-['ocean']
-
-'ocean.constant.melt_rate' = 0.0
-'ocean.models' = 'constant'
diff --git a/pism_terra/config/era_m2.toml b/pism_terra/config/era_m2.toml
deleted file mode 100644
index 9c61543..0000000
--- a/pism_terra/config/era_m2.toml
+++ /dev/null
@@ -1,110 +0,0 @@
-[run]
-
-exec = "/Users/andy/pism-dev-conda-petsc/bin/pism"
-mpi = "mpirun -np"
-ntasks = 8
-queue = "t2small"
-walltime = "12:00:00"
-
-[campaign]
-
-setup = "init"
-
-[climate]
-
-dataset="reanalysis-era5-land-monthly-means"
-
-[grid]
-
-resolution = "200m"
-'grid.Lbz' = 0
-'grid.Lz' = 2000
-'grid.Mbz' = 1
-'grid.Mz' = 101
-'grid.registration' = "center"
-
-['geometry']
-
-'geometry.front_retreat.use_cfl' = 'yes'
-'geometry.part_grid.enabled' = 'yes'
-'geometry.remove_icebergs' = 'yes'
-
-['input']
-
-'input.bootstrap' = "yes"
-
-[time]
-
-'time.start' = "1981-10-01"
-'time.end' = "2001-01-01"
-'time.calendar' = "standard"
-'time.reference_date' = "1980-01-01"
-'time_stepping.adaptive_ratio' = 250
-'time_stepping.skip.enabled' = "yes"
-'time_stepping.skip.max' = 100
-
-[reporting]
-
-'output.format' = "netcdf4_parallel"
-'output.size' = "medium"
-'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf,bmelt"
-'output.scalar.times' = "daily"
-'output.spatial.file' = "none"
-'output.spatial.times' = "monthly"
-'output.spatial.vars' = "climatic_mass_balance,thk,velsurf_mag,velbase_mag,ice_mass,mass_fluxes,mask,usurf"
-'output.checkpoint.interval' = 24
-
-[energy]
-
-model = "enthalpy"
-
-[energy.options]
-
-none = { 'energy.model' = "none", 'flow_law.isothermal_Glen.ice_softness' = 1.0e-24, 'stress_balance.blatter.flow_law' = "isothermal_glen", 'stress_balance.sia.flow_law' = "isothermal_glen", 'stress_balance.ssa.flow_law' = "isothermal_glen"}
-enthalpy = { 'energy.model' = "enthalpy", 'flow_law.isothermal_Glen.ice_softness' = 1.0e-24, 'stress_balance.blatter.flow_law' = "gpbld", 'stress_balance.sia.flow_law' = "gpbld", 'stress_balance.ssa.flow_law' = "gpbld"}
-
-[stress_balance]
-
-'stress_balance.model' = 'blatter'
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
-'stress_balance.blatter.use_eta_transform' = 'yes'
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.surface_gradient_method' = 'eta'
-'stress_balance.sia.max_diffusivity' = 100000.0
-
-[stress_balance.options]
-
-blatter = {'stress_balance.model' = "blatter", bp_ksp_monitor = "", bp_ksp_view_singularvalues = "", bp_snes_monitor_ratio = "", bp_pc_type = "mg", bp_mg_levels_ksp_type = "richardson", bp_mg_levels_pc_type ="sor", bp_mg_coarse_ksp_type = "preonly", bp_mg_coarse_pc_type = "lu", bp_pc_mg_levels = 3, bp_pc_type mg = "", bp_snes_ksp_ew = 1, bp_snes_ksp_ew_version = 3}
-
-hybrid = {'stress_balance.model' = "ssa+sia", 'stress_balance.ssa.method' = "fd"}
-
-[iceflow]
-
-'basal_resistance.pseudo_plastic.q' = 0.75
-'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
-'basal_resistance.pseudo_plastic.enabled' = 'yes'
-'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
-'basal_yield_stress.model' = 'mohr_coulomb'
-'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
-'stress_balance.blatter.enhancement_factor' = 2.0
-'stress_balance.sia.enhancement_factor' = 2.0
-
-['surface']
-
-'surface.models' = 'pdd,forcing'
-
-['calving']
-
-'calving.methods' = 'float_kill'
-
-['atmosphere']
-
-'atmosphere.models' = "given,elevation_change"
-'atmosphere.given.file' = "none"
-'atmosphere.elevation_change.temperature_lapse_rate' = 6
-
-['ocean']
-
-'ocean.constant.melt_rate' = 0.0
-'ocean.models' = 'constant'
diff --git a/pism_terra/config/ismip7_greenland.toml b/pism_terra/config/ismip7_greenland.toml
index bb517b8..cc05b34 100644
--- a/pism_terra/config/ismip7_greenland.toml
+++ b/pism_terra/config/ismip7_greenland.toml
@@ -1,18 +1,232 @@
-ismip7_to_pism = {"acabf" = "climatic_mass_balance", "runoff" = "water_input_rate", "tas" = "ice_surface_temp", "TF" = "theta_ocean", "geothermal_heat_flux1" = "bheatflx"}
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.experiment' = "ssp585"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+pathway = "ssp585"
+version = "v2"
+start_year = "1978"
+end_year = "2015"
gcms = ["CESM2-WACCM"]
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g2400m_id_BAYES-MEDIAN_1980-1-1_1985-01-01.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1985-01-01"
+'time.end' = "2300-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 37
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "lithk,orog,topg,hfgeoubed,acabf,libmassbfgr,libmassbffl,dlithkdt,velsurf,zvelsurf,velbase,zvelbase,velmean,litemptop,litempbotgr,litempbotfl,strbasemag,licalvf,lifmassbf,sftgif,sftgrf,sftflf"
+'output.scalar.times' = "monthly"
+'output.scalar.file' = "none"
+'output.scalar.variables' = "lim,limnsw,iareagr,iareafl,tendacabf,tendlibmassbf,tendlibmassbffl,tendlicalvf,tendlifmassbf"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP6' = "yes"
+
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 101
+'grid.Mz' = 201
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "th"
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="none"
+
+[bed_deformation.options.none]
-[domain]
+'bed_deformation.model' = "none"
-x_bounds = [-720500.0, 961500.0]
-y_bounds = [-3450500.0, -569500.0]
-resolution = "1000m"
+[bed_deformation.options.lc]
-[pathway]
+'bed_deformation.model' = "lc"
-historical = {"start_year" = 1975, "end_year" = 2015, version = 1}
-ssp585 = {"start_year" = 2015, "end_year" = 2299, version = 1 }
+[solver]
-[forcing]
+[solver.forward]
-climate = {fields = ["acabf", "runoff", "tas"], short_hand = "SDBN1"}
-ocean = {fields = ["TF"], short_hand = "none"}
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_c001.toml b/pism_terra/config/ismip7_greenland_c001.toml
new file mode 100644
index 0000000..031116f
--- /dev/null
+++ b/pism_terra/config/ismip7_greenland_c001.toml
@@ -0,0 +1,270 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.counter' = "C001"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+projection_start_year = "2015"
+projection_end_year = "2300"
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_HO.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "ismip6"
+'output.scalar.times' = "monthly"
+'output.scalar.file' = "yearly"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP' = "yes"
+'output.fill_value' = 9.969209968386869e+36
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 51
+'grid.Mz' = 101
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = "yes"
+'stress_balance.calving_front_stress_bc' = "yes"
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'basal_yield_stress.slippery_grounding_lines' = "yes"
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.thickness_calving.threshold' = 100
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "picop_sgd"
+
+[ocean.options.pico]
+
+'ocean.models' = "pico"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop_sgd]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "yes"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+'frontal_melt.include_floating_ice' = "no"
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_c001_ho.toml b/pism_terra/config/ismip7_greenland_c001_ho.toml
new file mode 100644
index 0000000..56d3cca
--- /dev/null
+++ b/pism_terra/config/ismip7_greenland_c001_ho.toml
@@ -0,0 +1,274 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.counter' = "C001"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+projection_start_year = "2015"
+projection_end_year = "2300"
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_HO.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "topg,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,ice_mass,grounding_line_flux,dHdt,bmelt,frontal_melt_rate,bmelt"
+'output.scalar.times' = "monthly"
+'output.scalar.file' = "none"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP' = "no"
+'output.fill_value' = 9.969209968386869e+36
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 51
+'grid.Mz' = 101
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = "yes"
+'stress_balance.calving_front_stress_bc' = "yes"
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'basal_yield_stress.slippery_grounding_lines' = "yes"
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.thickness_calving.threshold' = 50
+'calving.vonmises_calving.sigma_max' = 5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "picop_sgd"
+
+[ocean.options.pico]
+
+'ocean.models' = "pico"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+
+[ocean.options.picop]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "no"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.picop_sgd]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "yes"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+'frontal_melt.include_floating_ice' = "no"
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_c001_hybrid.toml b/pism_terra/config/ismip7_greenland_c001_hybrid.toml
new file mode 100644
index 0000000..92f3904
--- /dev/null
+++ b/pism_terra/config/ismip7_greenland_c001_hybrid.toml
@@ -0,0 +1,274 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.counter' = "C001"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+projection_start_year = "2015"
+projection_end_year = "2300"
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_HYBRID.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "topg,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,ice_mass,grounding_line_flux,dHdt,bmelt,frontal_melt_rate,bmelt"
+'output.scalar.times' = "monthly"
+'output.scalar.file' = "none"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP' = "no"
+'output.fill_value' = 9.969209968386869e+36
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 51
+'grid.Mz' = 101
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'hybrid'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = "yes"
+'stress_balance.calving_front_stress_bc' = "yes"
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'basal_yield_stress.slippery_grounding_lines' = "yes"
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.thickness_calving.threshold' = 50
+'calving.vonmises_calving.sigma_max' = 5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "picop_sgd"
+
+[ocean.options.pico]
+
+'ocean.models' = "pico"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+
+[ocean.options.picop]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "no"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.picop_sgd]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "yes"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+'frontal_melt.include_floating_ice' = "no"
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_c002.toml b/pism_terra/config/ismip7_greenland_c002.toml
new file mode 100644
index 0000000..11719be
--- /dev/null
+++ b/pism_terra/config/ismip7_greenland_c002.toml
@@ -0,0 +1,269 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.counter' = "C002"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+projection_start_year = "2015"
+projection_end_year = "2300"
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_1980-01-01_1980-07-01.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "ismip6"
+'output.scalar.times' = "yearly"
+'output.scalar.file' = "none"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP' = "yes"
+'output.fill_value' = 9.969209968386869e+36
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 51
+'grid.Mz' = 101
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = "yes"
+'stress_balance.calving_front_stress_bc' = "yes"
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'basal_yield_stress.slippery_grounding_lines' = "yes"
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.thickness_calving.threshold' = 100
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "picop_sgd"
+
+[ocean.options.pico]
+
+'ocean.models' = "pico"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop_sgd]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "yes"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_c003.toml b/pism_terra/config/ismip7_greenland_c003.toml
new file mode 100644
index 0000000..0021ebe
--- /dev/null
+++ b/pism_terra/config/ismip7_greenland_c003.toml
@@ -0,0 +1,269 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.counter' = "C003"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+projection_start_year = "2015"
+projection_end_year = "2300"
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_1980-01-01_1980-07-01.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "ismip6"
+'output.scalar.times' = "yearly"
+'output.scalar.file' = "none"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP' = "yes"
+'output.fill_value' = 9.969209968386869e+36
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 51
+'grid.Mz' = 101
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = "yes"
+'stress_balance.calving_front_stress_bc' = "yes"
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'basal_yield_stress.slippery_grounding_lines' = "yes"
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.thickness_calving.threshold' = 100
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "picop_sgd"
+
+[ocean.options.pico]
+
+'ocean.models' = "pico"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop_sgd]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "yes"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_c004.toml b/pism_terra/config/ismip7_greenland_c004.toml
new file mode 100644
index 0000000..6d582ec
--- /dev/null
+++ b/pism_terra/config/ismip7_greenland_c004.toml
@@ -0,0 +1,269 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.counter' = "C004"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+projection_start_year = "2015"
+projection_end_year = "2300"
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_1980-01-01_1980-07-01.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "ismip6"
+'output.scalar.times' = "yearly"
+'output.scalar.file' = "none"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP' = "yes"
+'output.fill_value' = 9.969209968386869e+36
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 51
+'grid.Mz' = 101
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = "yes"
+'stress_balance.calving_front_stress_bc' = "yes"
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'basal_yield_stress.slippery_grounding_lines' = "yes"
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.thickness_calving.threshold' = 100
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "picop_sgd"
+
+[ocean.options.pico]
+
+'ocean.models' = "pico"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop_sgd]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "yes"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_c005.toml b/pism_terra/config/ismip7_greenland_c005.toml
new file mode 100644
index 0000000..f59ee4c
--- /dev/null
+++ b/pism_terra/config/ismip7_greenland_c005.toml
@@ -0,0 +1,269 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.counter' = "C005"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+projection_start_year = "2015"
+projection_end_year = "2300"
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_1980-01-01_1980-07-01.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "ismip6"
+'output.scalar.times' = "yearly"
+'output.scalar.file' = "none"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP' = "yes"
+'output.fill_value' = 9.969209968386869e+36
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 51
+'grid.Mz' = 101
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = "yes"
+'stress_balance.calving_front_stress_bc' = "yes"
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'basal_yield_stress.slippery_grounding_lines' = "yes"
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.thickness_calving.threshold' = 100
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "picop_sgd"
+
+[ocean.options.pico]
+
+'ocean.models' = "pico"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop_sgd]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "yes"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_c006.toml b/pism_terra/config/ismip7_greenland_c006.toml
new file mode 100644
index 0000000..407f8b3
--- /dev/null
+++ b/pism_terra/config/ismip7_greenland_c006.toml
@@ -0,0 +1,269 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.counter' = "C006"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+projection_start_year = "2015"
+projection_end_year = "2300"
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_1980-01-01_1980-07-01.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "ismip6"
+'output.scalar.times' = "yearly"
+'output.scalar.file' = "none"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP' = "yes"
+'output.fill_value' = 9.969209968386869e+36
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 51
+'grid.Mz' = 101
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = "yes"
+'stress_balance.calving_front_stress_bc' = "yes"
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'basal_yield_stress.slippery_grounding_lines' = "yes"
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.thickness_calving.threshold' = 100
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "picop_sgd"
+
+[ocean.options.pico]
+
+'ocean.models' = "pico"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop_sgd]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "yes"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_c007.toml b/pism_terra/config/ismip7_greenland_c007.toml
new file mode 100644
index 0000000..82bbc6a
--- /dev/null
+++ b/pism_terra/config/ismip7_greenland_c007.toml
@@ -0,0 +1,269 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.counter' = "C007"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+projection_start_year = "2015"
+projection_end_year = "2300"
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_1980-01-01_1980-07-01.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "ismip6"
+'output.scalar.times' = "yearly"
+'output.scalar.file' = "none"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP' = "yes"
+'output.fill_value' = 9.969209968386869e+36
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 51
+'grid.Mz' = 101
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = "yes"
+'stress_balance.calving_front_stress_bc' = "yes"
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'basal_yield_stress.slippery_grounding_lines' = "yes"
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.thickness_calving.threshold' = 100
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "picop_sgd"
+
+[ocean.options.pico]
+
+'ocean.models' = "pico"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop_sgd]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "yes"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_c008.toml b/pism_terra/config/ismip7_greenland_c008.toml
new file mode 100644
index 0000000..accfd9b
--- /dev/null
+++ b/pism_terra/config/ismip7_greenland_c008.toml
@@ -0,0 +1,269 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.counter' = "C008"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+projection_start_year = "2015"
+projection_end_year = "2300"
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_1980-01-01_1980-07-01.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "ismip6"
+'output.scalar.times' = "yearly"
+'output.scalar.file' = "none"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP' = "yes"
+'output.fill_value' = 9.969209968386869e+36
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 51
+'grid.Mz' = 101
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = "yes"
+'stress_balance.calving_front_stress_bc' = "yes"
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'basal_yield_stress.slippery_grounding_lines' = "yes"
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.thickness_calving.threshold' = 100
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "picop_sgd"
+
+[ocean.options.pico]
+
+'ocean.models' = "pico"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+
+[ocean.options.picop_sgd]
+
+'ocean.models' = "picop"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 3
+'ocean.pico.overturning_coefficent' = 1e6
+'ocean.pico.temperature_as_thermal_forcing' = "yes"
+'ocean.picop.add_fresh_water_melt' = "yes"
+'ocean.picop.power_alpha' = 0.54
+'ocean.picop.power_beta' = 1.18
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_historical.toml b/pism_terra/config/ismip7_greenland_historical.toml
index b9a19da..bb70ea5 100644
--- a/pism_terra/config/ismip7_greenland_historical.toml
+++ b/pism_terra/config/ismip7_greenland_historical.toml
@@ -1,45 +1,40 @@
-[run]
-
-executable = "pism"
-mpi = "mpirun -np {{ ntasks }}"
-ntasks = 80
-
-[job]
-
-queue = "t2small"
-walltime = "48:00:00"
-nodes = 2
-
[run_info]
'run_info.institution' = "University of Alaska Fairbanks"
'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.counter' = "C001"
+'run_info.domain' = "GrIS"
+'run_info.set' = "CORE"
[campaign]
bucket = "pism-cloud-data"
-prefix = "ismip7_greenland_input"
-pathway = "historical"
-version = "1"
-start_year = "1975"
-end_year = "2015"
-gcm = "CESM2-WACCM"
-boot_file = "boot_g1000m_GreenlandObsISMIP7-v1.3.nc"
-grid_file = "ismip7_greenland_grid.nc"
-regrid_file = "g600m_id_BAYES-MEDIAN_1980-1-1_1984-12-31.nc"
-retreat_file = "pism_g1000m_frontretreat_calfin_1972_2019_ME.nc"
-heatflux_file = "heatflux_GreenlandObsISMIP7-v1.3.nc"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+projection_start_year = "2015"
+projection_end_year = "2300"
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_1980-01-01_1981-01-01.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
[time]
-'time.start' = "1980-01-01"
-'time.end' = "2015-01-01"
-'time.calendar' = "standard"
-'time.reference_date' = "1978-01-01"
+'time.start' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
[time_stepping]
-'time_stepping.adaptive_ratio' = 250
'time_stepping.skip.enabled' = "yes"
'time_stepping.skip.max' = 100
@@ -51,32 +46,33 @@ heatflux_file = "heatflux_GreenlandObsISMIP7-v1.3.nc"
['input']
+'input.file' = "none"
'input.bootstrap' = "yes"
'input.forcing.time_extrapolation' = "yes"
-'input.forcing.buffer_size' = 37
+'input.forcing.buffer_size' = 24
'input.regrid.file' = "none"
-'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
[reporting]
-'output.format' = "netcdf4_serial"
+'output.format' = "netcdf4_parallel"
'output.compression_level' = 2
'output.size' = "medium"
'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
-'output.extra.file' = "none"
-'output.extra.times' = "monthly"
-'output.extra.vars' = "lithk,orog,topg,hfgeoubed,acabf,libmassbfgr,libmassbffl,dlithkdt,velsurf,zvelsurf,velbase,zvelbase,velmean,litemptop,litempbotgr,litempbotfl,strbasemag,licalvf,lifmassbf,sftgif,sftgrf,sftflf"
-'output.timeseries.filename' = "none"
-'output.timeseries.times' = "monthly"
-'output.timeseries.variables' = "lim,limnsw,iareagr,iareafl,tendacabf,tendlibmassbf,tendlibmassbffl,tendlicalvf,tendlifmassbf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes,mass,mass_fluxes,grounding_line_flux,dHdt,bmelt,frontal_melt_rate,subglacial_water_flux,bmelt"
+'output.scalar.times' = "monthly"
+'output.scalar.file' = "none"
'output.checkpoint.interval' = 24
-'output.extra.stop_missing' = 'no'
-'output.ISMIP6' = "yes"
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP6' = "no"
[grid]
-resolution = "1000m"
+resolution = "900m"
+'grid.file' = "none"
'grid.Lbz' = 2000
'grid.Lz' = 4000
'grid.Mbz' = 101
@@ -98,28 +94,16 @@ model = "enthalpy"
[stress_balance]
-model = 'hybrid'
+model = 'blatter'
[stress_balance.options.blatter]
'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
'stress_balance.calving_front_stress_bc' = 'yes'
'stress_balance.sia.max_diffusivity' = 100000.0
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
+'time_stepping.adaptive_ratio' = 1000
[stress_balance.options.hybrid]
@@ -145,7 +129,14 @@ bp_ksp_rtol = 1e-3
[surface]
-model = "given"
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
[surface.options.given]
@@ -168,21 +159,83 @@ model = "given"
[ocean]
+model = "pico"
+
+[ocean.options.pico]
+
+'ocean.models' = "pico"
+'ocean.pico.file' = "none"
+'ocean.pico.continental_shelf_depth' = -280
+'ocean.pico.heat_exchange_coefficent' = 2e-5
+'ocean.pico.maximum_ice_rise_area' = 10000.0
+'ocean.pico.number_of_boxes' = 5
+'ocean.pico.overturning_coefficent' = 1e6
+
+[ocean.options.th]
+
'ocean.models' = "th"
'ocean.th.file' = "none"
-'ocean.th.clip_salinity' = "False"
+'ocean.th.clip_salinity' = "yes"
[frontal_melt]
-model = "off"
+model = "routing"
[frontal_melt.options.off]
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+
[hydrology]
-model = "null"
+model = "routing"
[hydrology.options.null]
'hydrology.model' = "null"
'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
diff --git a/pism_terra/config/ismip7_greenland_init.toml b/pism_terra/config/ismip7_greenland_init.toml
new file mode 100644
index 0000000..e998dac
--- /dev/null
+++ b/pism_terra/config/ismip7_greenland_init.toml
@@ -0,0 +1,270 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
+'run_info.group' = "UAF"
+'run_info.contact_email' = "aaschwanden@alaska.edu"
+'run_info.contact_name' = "Andy Aschwanden"
+'run_info.model' = "PISM"
+'run_info.ism' = "PISM"
+'run_info.experiment' = "CALIB"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+version = "v2"
+historical_start_year = "1978"
+historical_end_year = "2014"
+gcms =["CESM2-WACCM"]
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+grid_file = "pism_bedmachine_greenland_grid.nc"
+regrid_file = "g900m_id_BAYES-MEDIAN_1980-01-01_1981-01-01.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+obs_file = "obs_g900m_GreenlandObsISMIP7-v1.3.nc"
+
+[time]
+
+'time.start' = "1980-01-01"
+'time.end' = "1985-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "1850-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.file' = "none"
+'input.bootstrap' = "yes"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+'input.regrid.file' = "none"
+'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume,thk,uvel_sigma,vvel_sigma"
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level' = 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes,mass_fluxes,grounding_line_flux,dHdt,bmelt,frontal_melt_rate,subglacial_water_flux"
+'output.scalar.times' = "monthly"
+'output.scalar.file' = "none"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+'output.ISMIP6' = "no"
+
+
+[grid]
+
+resolution = "900m"
+'grid.file' = "none"
+'grid.Lbz' = 2000
+'grid.Lz' = 4000
+'grid.Mbz' = 101
+'grid.Mz' = 201
+'grid.registration' = "center"
+'grid.allow_extrapolation' = "yes"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 1000
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_resistance.pseudo_plastic.q' = 0.7508221
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
+'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
+'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
+'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
+'basal_yield_stress.slippery_grounding_lines' = "yes"
+'stress_balance.blatter.enhancement_factor' = 2.608046
+'stress_balance.sia.enhancement_factor' = 2.608046
+'stress_balance.ssa.Glen_exponent' = 3.309718
+
+[surface]
+
+model = "ismip7"
+
+[surface.options.ismip7]
+
+'surface.models' = "ismip7"
+'surface.ismip7.file' = "none"
+'surface.ismip7.gradient.file' = "none"
+'surface.ismip7.reference.file' = "none"
+
+[surface.options.given]
+
+'surface.models' = "given"
+'surface.given.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+
+[ocean]
+
+model = "th"
+
+[ocean.options.th]
+
+'ocean.models' = "th"
+'ocean.th.file' = "none"
+'ocean.th.clip_salinity' = "yes"
+
+[frontal_melt]
+
+model = "routing"
+
+[frontal_melt.options.off]
+
+[frontal_melt.options.routing]
+
+# multiply default values by 1.63
+# as recommended by ISMIP7
+
+'frontal_melt.models' = "routing"
+'frontal_melt.routing.file' = "none"
+'frontal_melt.routing.parameter_a' = 4.89e-4
+'frontal_melt.routing.parameter_b' = 0.25
+
+[hydrology]
+
+model = "routing"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "yes"
+
+[hydrology.options.routing]
+
+'hydrology.model' = "routing"
+'hydrology.surface_input_from_runoff' = "yes"
+
+[bed_deformation]
+
+model="lc"
+
+[bed_deformation.options.none]
+
+'bed_deformation.model' = "none"
+
+[bed_deformation.options.lc]
+
+'bed_deformation.model' = "lc"
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_monitor = ""
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
+
+[solver.inverse]
+
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 0
+bp_snes_ksp_ew_version = 3
+bp_snes_monitor = ""
+bp_snes_rtol = 1e-3
+inv_adj_ksp_type = "gmres"
+inv_adj_pc_type = "jacobi"
+
+[inverse]
+
+'inverse.adjoint.method' = "approximate"
+'inverse.design.cH1' = 1
+'inverse.design.cL2' = 0
+'inverse.file' = "none"
+'inverse.stress_balance.velocity_scale' = 1e2
+'inverse.stress_balance.length_scale' = 50e3
+'inverse.design.param' = "exp"
+'inverse.huber.delta' = 10e3
+'inverse.max_iterations' = 250
+'inverse.state_func' = "huber"
+'inverse.stress_balance.method' = "tikhonov_blmvm"
+'inverse.stress_balance.tauc_min' = 1e4
+'inverse.stress_balance.tauc_max' = 5e7
+'inverse.tikhonov.penalty_weight' = 10
+'inverse.tikhonov.atol' = 1e-7
+'inverse.use_zeta_fixed_mask' = "yes"
+'inverse.use_design_prior' = "no"
diff --git a/pism_terra/config/kitp_greenland.toml b/pism_terra/config/kitp_greenland.toml
index 8cff4b6..6fa4ec7 100644
--- a/pism_terra/config/kitp_greenland.toml
+++ b/pism_terra/config/kitp_greenland.toml
@@ -1,16 +1,3 @@
-[run]
-
-executable = "pism"
-mpi = "mpirun -np {{ ntasks }}"
-ntasks = 8
-writer = ""
-
-[job]
-
-queue = "t2small"
-walltime = "48:00:00"
-nodes = 1
-
[run_info]
'run_info.institution' = "University of Alaska Fairbanks"
@@ -18,22 +5,32 @@ nodes = 1
[campaign]
-bucket = "pism-cloud-data"
-prefix = "kitp/input"
-version = "v1"
-gcms = "CESM1-WACCM-SC"
-present_day_forcings = ["pdSST-pdSIC", "pdSST-pdSICSIT", "pa-pdSIC-ext"]
-future_forcings = ["futSST-pdSIC", "pdSST-futArcSIC", "pa-futArcSIC-ext"]
boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+bucket = "pism-cloud-data"
+climatology = "HIRHAM5-ERA5_YMM_1990_2019"
grid_file = "ismip7_greenland_grid.nc"
-regrid_file = "g900m_id_BAYES-MEDIAN_2007-01-01_2010-01-01.nc"
heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+ocean_file = "ocean_forcing_228.0_10.0_69.0_80.0.nc"
outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+prefix = "kitp/input"
+regrid_file = "g900m_id_BAYES-MEDIAN_2007-01-01_2010-01-01.nc"
+retreat_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+version = "v4"
+
+[campaign.gcms]
+
+'AWI-CM-1-1-MR' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CESM1-WACCM-SC' = [["pdSST-futArcSICSIT", "pdSST-pdSICSIT"], ["pa-futArcSIC-ext", "pa-pdSIC-ext"], ["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CESM2' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CNRM-CM6-1' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CanESM5' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'HadGEM3-GC31-MM' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'IPSL-CM6A-LR' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
[time]
'time.start' = "0001-01-01"
-'time.end' = "0300-01-01"
+'time.end' = "0301-01-01"
'time.calendar' = "365_day"
'time.reference_date' = "0001-01-01"
@@ -48,10 +45,12 @@ outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
'geometry.front_retreat.use_cfl' = 'yes'
'geometry.part_grid.enabled' = 'yes'
'geometry.remove_icebergs' = 'yes'
+'geometry.front_retreat.prescribed.file' = "none"
['input']
'input.bootstrap' = "yes"
+'input.file' = "none"
'input.forcing.time_extrapolation' = "yes"
'input.forcing.buffer_size' = 37
'input.regrid.file' = "none"
@@ -65,22 +64,22 @@ outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
'output.spatial.file' = "none"
'output.spatial.times' = "yearly"
-'output.spatial.vars' = "bmelt,effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,ice_mass,mass_fluxes"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,ice_mass,mass_fluxes,grounding_line_flux,pdd_fluxes"
'output.scalar.file' = "none"
'output.scalar.times' = "monthly"
'output.checkpoint.interval' = 24
-'output.extra.stop_missing' = 'no'
+'output.spatial.stop_missing' = 'no'
'output.ISMIP6' = "no"
'output.experiment_id_max_length' = 100
[grid]
resolution = "900m"
+'grid.file' = "none"
'grid.Lbz' = 2000
'grid.Lz' = 4000
'grid.Mbz' = 51
'grid.Mz' = 101
-'grid.registration' = "center"
'grid.allow_extrapolation' = "yes"
[energy]
@@ -102,23 +101,10 @@ model = 'hybrid'
[stress_balance.options.blatter]
'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
'stress_balance.calving_front_stress_bc' = 'yes'
'stress_balance.sia.max_diffusivity' = 1000000.0
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
[stress_balance.options.hybrid]
@@ -127,6 +113,12 @@ bp_ksp_rtol = 1e-3
'stress_balance.calving_front_stress_bc' = 'yes'
'stress_balance.sia.max_diffusivity' = 100000.0
+[stress_balance.options.sia]
+
+'stress_balance.model' = "sia"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
[iceflow]
'basal_resistance.pseudo_plastic.enabled' = 'yes'
@@ -144,28 +136,63 @@ bp_ksp_rtol = 1e-3
[surface]
-model = "pdd"
+model = "debm"
[surface.options.pdd]
'surface.models' = "pdd"
+'surface.pdd.std_dev.file' = "none"
+'surface.pdd.std_dev.periodic' = "yes"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.albedo_input.file' = "none"
+'surface.debm_simple.albedo_input.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 25
+'surface.debm_simple.c2' = -103
+'surface.debm_simple.refreeze' = 0.50
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 271.65
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.0
[calving]
-'calving.methods' = 'vonmises_calving'
-'calving.vonmises_calving.sigma_max' = 7.5e5
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.vonmises_calving.sigma_max' = 4.0e5
+'calving.thickness_calving.threshold' = 100.0
[atmosphere]
-model = "given"
+model = "given_elevation_change"
[atmosphere.options.given]
'atmosphere.models' = "given"
'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[atmosphere.options.given_elevation_change]
+
+'atmosphere.models' = "given,elevation_change"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+'atmosphere.elevation_change.temperature_lapse_rate' = 6
+'atmosphere.elevation_change.file' = "none"
[ocean]
+model = "given"
+
+[ocean.options.given]
+
+'ocean.models' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
[hydrology]
model = "null"
@@ -195,3 +222,21 @@ model = "off"
'frontal_melt.routing.parameter_b' = 0.15
'frontal_melt.routing.power_alpha' = 0.39
'frontal_melt.routing.power_beta' = 1.18
+
+[solver.forward]
+bp_ksp_monitor = ""
+bp_ksp_view_singularvalues = ""
+bp_snes_monitor_ratio = ""
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/test_kitp_greenland.toml b/pism_terra/config/kitp_greenland_1year.toml
similarity index 51%
rename from pism_terra/config/test_kitp_greenland.toml
rename to pism_terra/config/kitp_greenland_1year.toml
index 31da0ef..71942ec 100644
--- a/pism_terra/config/test_kitp_greenland.toml
+++ b/pism_terra/config/kitp_greenland_1year.toml
@@ -1,16 +1,3 @@
-[run]
-
-executable = "pism"
-mpi = "mpirun -np {{ ntasks }}"
-ntasks = 8
-writer = ""
-
-[job]
-
-queue = "t2small"
-walltime = "4:00:00"
-nodes = 1
-
[run_info]
'run_info.institution' = "University of Alaska Fairbanks"
@@ -18,22 +5,32 @@ nodes = 1
[campaign]
+boot_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
bucket = "pism-cloud-data"
-prefix = "kitp/input_testing"
-version = "v1"
-gcms = "CESM1-WACCM-SC"
-present_day_forcings = ["pdSST-pdSIC", "pdSST-pdSICSIT", "pa-pdSIC-ext"]
-future_forcings = ["futSST-pdSIC", "pdSST-futArcSIC", "pa-futArcSIC-ext"]
-boot_file = "boot_g2400m_GreenlandObsISMIP7-v1.3.nc"
+climatology = "HIRHAM5-ERA5_YMM_1990_2019"
grid_file = "ismip7_greenland_grid.nc"
-regrid_file = "g2400m_id_BAYES-MEDIAN_2007-01-01_2010-01-01.nc"
-heatflux_file = "heatflux_g2400m_GreenlandObsISMIP7-v1.3.nc"
+heatflux_file = "heatflux_g900m_GreenlandObsISMIP7-v1.3.nc"
+ocean_file = "ocean_forcing_228.0_10.0_69.0_80.0.nc"
outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
+prefix = "kitp/input"
+regrid_file = "g900m_id_BAYES-MEDIAN_2007-01-01_2010-01-01.nc"
+retreat_file = "boot_g900m_GreenlandObsISMIP7-v1.3.nc"
+version = "v4"
+
+[campaign.gcms]
+
+'AWI-CM-1-1-MR' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CESM1-WACCM-SC' = [["pdSST-futArcSICSIT", "pdSST-pdSICSIT"], ["pa-futArcSIC-ext", "pa-pdSIC-ext"], ["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CESM2' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CNRM-CM6-1' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CanESM5' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'HadGEM3-GC31-MM' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'IPSL-CM6A-LR' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
[time]
'time.start' = "0001-01-01"
-'time.end' = "0005-01-01"
+'time.end' = "0003-01-01"
'time.calendar' = "365_day"
'time.reference_date' = "0001-01-01"
@@ -48,10 +45,12 @@ outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
'geometry.front_retreat.use_cfl' = 'yes'
'geometry.part_grid.enabled' = 'yes'
'geometry.remove_icebergs' = 'yes'
+'geometry.front_retreat.prescribed.file' = "none"
['input']
'input.bootstrap' = "yes"
+'input.file' = "none"
'input.forcing.time_extrapolation' = "yes"
'input.forcing.buffer_size' = 37
'input.regrid.file' = "none"
@@ -59,28 +58,28 @@ outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
[reporting]
-'output.format' = "netcdf4_serial"
+'output.format' = "netcdf4_parallel"
'output.compression_level' = 2
'output.size' = "medium"
'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
'output.spatial.file' = "none"
'output.spatial.times' = "monthly"
-'output.spatial.vars' = "bmelt,effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,tauc,tillwat,diffusivity,ice_mass,mass_fluxes"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,ice_mass,mass_fluxes,grounding_line_flux,pdd_fluxes"
'output.scalar.file' = "none"
'output.scalar.times' = "monthly"
'output.checkpoint.interval' = 24
-'output.extra.stop_missing' = 'no'
+'output.spatial.stop_missing' = 'no'
'output.ISMIP6' = "no"
'output.experiment_id_max_length' = 100
[grid]
-'resolution' = "2400m"
+resolution = "1200m"
+'grid.file' = "none"
'grid.Lbz' = 2000
'grid.Lz' = 4000
'grid.Mbz' = 51
'grid.Mz' = 101
-'grid.registration' = "center"
'grid.allow_extrapolation' = "yes"
[energy]
@@ -102,23 +101,10 @@ model = 'hybrid'
[stress_balance.options.blatter]
'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
'stress_balance.calving_front_stress_bc' = 'yes'
'stress_balance.sia.max_diffusivity' = 1000000.0
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
[stress_balance.options.hybrid]
@@ -127,6 +113,12 @@ bp_ksp_rtol = 1e-3
'stress_balance.calving_front_stress_bc' = 'yes'
'stress_balance.sia.max_diffusivity' = 100000.0
+[stress_balance.options.sia]
+
+'stress_balance.model' = "sia"
+'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
[iceflow]
'basal_resistance.pseudo_plastic.enabled' = 'yes'
@@ -144,28 +136,63 @@ bp_ksp_rtol = 1e-3
[surface]
-model = "pdd"
+model = "debm"
[surface.options.pdd]
'surface.models' = "pdd"
+'surface.pdd.std_dev.file' = "none"
+'surface.pdd.std_dev.periodic' = "yes"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.albedo_input.file' = "none"
+'surface.debm_simple.albedo_input.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 42
+'surface.debm_simple.c2' = -128
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 272.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
[calving]
-'calving.methods' = 'vonmises_calving'
-'calving.vonmises_calving.sigma_max' = 7.5e5
+'calving.methods' = 'vonmises_calving,thickness_calving'
+'calving.vonmises_calving.sigma_max' = 4.0e5
+'calving.thickness_calving.threshold' = 100.0
[atmosphere]
-model = "given"
+model = "given_elevation_change"
[atmosphere.options.given]
'atmosphere.models' = "given"
'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[atmosphere.options.given_elevation_change]
+
+'atmosphere.models' = "given,elevation_change"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+'atmosphere.elevation_change.temperature_lapse_rate' = 6
+'atmosphere.elevation_change.file' = "none"
[ocean]
+model = "given"
+
+[ocean.options.given]
+
+'ocean.models' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
[hydrology]
model = "null"
@@ -195,3 +222,21 @@ model = "off"
'frontal_melt.routing.parameter_b' = 0.15
'frontal_melt.routing.power_alpha' = 0.39
'frontal_melt.routing.power_beta' = 1.18
+
+[solver.forward]
+bp_ksp_monitor = ""
+bp_ksp_view_singularvalues = ""
+bp_snes_monitor_ratio = ""
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/rgi.toml b/pism_terra/config/rgi.toml
deleted file mode 100644
index effd935..0000000
--- a/pism_terra/config/rgi.toml
+++ /dev/null
@@ -1,21 +0,0 @@
-[regions]
-
-1 = "alaska"
-2 = "western_canada_usa"
-3 = "arctic_canada_north"
-4 = "arctic_canada_south"
-5 = "greenland_periphery"
-6 = "iceland"
-7 = "svalbard_jan_mayen"
-8 = "scandinavia"
-9 = "russian_arctic"
-10 = "north_asia"
-11 = "central_europe"
-12 = "caucasus_middle_east"
-13 = "central_asia"
-14 = "south_asia_west"
-15 = "south_asia_east"
-16 = "low_latitudes"
-17 = "southern_andes"
-18 = "new_zealand"
-19 = "subantarctic_antarctic_islands"
diff --git a/pism_terra/config/era5_chinook.toml b/pism_terra/config/rgi_calib.toml
similarity index 74%
rename from pism_terra/config/era5_chinook.toml
rename to pism_terra/config/rgi_calib.toml
index 53bfa01..f4aa792 100644
--- a/pism_terra/config/era5_chinook.toml
+++ b/pism_terra/config/rgi_calib.toml
@@ -1,15 +1,3 @@
-[run]
-
-executable = "pism"
-mpi = "mpirun -np {{ ntasks }}"
-ntasks = 40
-
-[job]
-
-queue = "t2small"
-walltime = "48:00:00"
-nodes = 1
-
[run_info]
'run_info.institution' = "University of Alaska Fairbanks"
@@ -19,23 +7,26 @@ nodes = 1
bucket = "pism-cloud-data"
climate = "era5"
-dem = "glo_30"
+dem = "glo_90"
ice_thickness = "maffezzoli"
+bathymetry = "gebco"
name = "Calibration"
-prefix = "glacier"
-rgi_file = "rgi_c.gpkg"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
velocity = "its_live"
+forcing_mask = "all"
[time]
-'time.start' = "1978-01-01"
-'time.end' = "2025-01-01"
+'time.start' = "1980-01-01"
+'time.end' = "2020-01-01"
'time.calendar' = "standard"
'time.reference_date' = "1978-01-01"
[time_stepping]
-'time_stepping.adaptive_ratio' = 250
+'time_stepping.adaptive_ratio' = 500
'time_stepping.skip.enabled' = "yes"
'time_stepping.skip.max' = 100
@@ -48,6 +39,7 @@ velocity = "its_live"
['input']
'input.bootstrap' = "yes"
+'input.file' = "none"
'input.forcing.time_extrapolation' = "yes"
'input.forcing.buffer_size' = 390
@@ -59,7 +51,7 @@ velocity = "its_live"
'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
'output.scalar.times' = "daily"
'output.spatial.file' = "none"
-'output.spatial.times' = "monthly"
+'output.spatial.times' = "yearly"
'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf"
'output.checkpoint.interval' = 24
'output.spatial.stop_missing' = 'no'
@@ -67,6 +59,7 @@ velocity = "its_live"
[grid]
resolution = "250m"
+'grid.file' = "none"
'grid.Lbz' = 0
'grid.Lz' = 2000
'grid.Mbz' = 1
@@ -99,31 +92,18 @@ model = 'blatter'
[stress_balance.options.blatter]
'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
'stress_balance.blatter.use_eta_transform' = 'yes'
-'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
'stress_balance.sia.surface_gradient_method' = 'eta'
'stress_balance.sia.max_diffusivity' = 100000.0
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
[stress_balance.options.hybrid]
'stress_balance.model' = "ssa+sia"
'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
'stress_balance.sia.surface_gradient_method' = 'eta'
'stress_balance.sia.max_diffusivity' = 100000.0
@@ -136,7 +116,10 @@ bp_ksp_rtol = 1e-3
'basal_yield_stress.model' = 'mohr_coulomb'
'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
[surface]
@@ -149,6 +132,7 @@ model = "pdd"
[surface.options.pdd]
'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.file' = "none"
[calving]
@@ -168,11 +152,47 @@ model = "given"
[ocean]
-'ocean.constant.melt_rate' = 0.0
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
[hydrology]
model = "null"
-[hydrology.options]
+[hydrology.options.null]
+
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[solver.forward]
+bp_ksp_monitor = ""
+bp_ksp_view_singularvalues = ""
+bp_snes_monitor_ratio = ""
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/rgi_carra2_maffezzoli.toml b/pism_terra/config/rgi_carra2_maffezzoli.toml
new file mode 100644
index 0000000..c1b998b
--- /dev/null
+++ b/pism_terra/config/rgi_carra2_maffezzoli.toml
@@ -0,0 +1,267 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Initialization Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "carra2"
+dem = "glo_90"
+ice_thickness = "maffezzoli"
+bathymetry = "gebco"
+heatflux = "lucazeau"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "none"
+
+[time]
+
+'time.start' = "1986-01-01"
+'time.end' = "2025-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "1980-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 500
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes,mass_fluxes,mass"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.file' = "none"
+'grid.Lbz' = 0
+'grid.Lz' = 1200
+'grid.Mbz' = 1
+'grid.Mz' = 61
+'grid.registration' = "center"
+
+[energy]
+
+model = "none"
+
+[energy.options.none]
+
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'none'
+
+[stress_balance.options.none]
+
+'stress_balance.model' = "none"
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 0.12
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 35
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "debm_enhanced"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd"
+'surface.pdd.std_dev.file' = "none"
+'surface.pdd.std_dev.periodic' = "yes"
+
+[surface.options.debm_simple]
+
+'surface.models' = "debm_simple"
+'surface.debm_simple.albedo_input.file' = "none"
+'surface.debm_simple.albedo_input.periodic' = "yes"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 25
+'surface.debm_simple.c2' = -103
+'surface.debm_simple.refreeze' = 0.50
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 271.65
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.0
+
+[surface.options.debm_enhanced]
+
+'surface.models' = "debm_enhanced"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 29
+'surface.debm_simple.c2' = -93
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 273.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[bed_deformation]
+
+model = "none"
+
+[bed_deformation.options.none]
+
+
+[inverse]
+
+'inverse.file' = "none"
+'inverse.stress_balance.velocity_scale' = 1e2
+'inverse.stress_balance.length_scale' = 1e3
+'inverse.design.cH1' = 1
+'inverse.design.cL2' = 0
+'inverse.max_iterations' = 50
+'inverse.design.param' = "exp"
+'inverse.stress_balance.method' = "tikhonov_blmvm"
+'inverse.stress_balance.tauc_min' = 1e4
+'inverse.stress_balance.tauc_max' = 5e7
+'inverse.tikhonov.penalty_weight' = 10
+'inverse.use_zeta_fixed_mask' = "yes"
+'inverse.adjoint.method' = "exact"
+'inverse.state_func' = "huber"
+'inverse.huber.delta' = 1e2
+
+[solver.forward]
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
+
+[solver.inverse]
+inv_adj_ksp_type = "gmres"
+inv_adj_pc_type = "jacobi"
+bp_ksp_rtol = 1e-4
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_mg_levels_ksp_type = "chebyshev"
+bp_pc_mg_levels = 3
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_snes_ksp_ew = 0
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-4
diff --git a/pism_terra/config/rgi_carra2_maffezzoli_forward.toml b/pism_terra/config/rgi_carra2_maffezzoli_forward.toml
new file mode 100644
index 0000000..7969f5f
--- /dev/null
+++ b/pism_terra/config/rgi_carra2_maffezzoli_forward.toml
@@ -0,0 +1,268 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Initialization Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "carra2"
+dem = "glo_90"
+ice_thickness = "maffezzoli"
+bathymetry = "gebco"
+heatflux = "lucazeau"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "none"
+
+[time]
+
+'time.start' = "1986-01-01"
+'time.end' = "2025-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "1980-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 500
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes,mass_fluxes,mass"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.file' = "none"
+'grid.Lbz' = 0
+'grid.Lz' = 1200
+'grid.Mbz' = 1
+'grid.Mz' = 61
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.none]
+
+'stress_balance.model' = "none"
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.ue_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 0.12
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 35
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "debm_enhanced"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd"
+'surface.pdd.std_dev.file' = "none"
+'surface.pdd.std_dev.periodic' = "yes"
+
+[surface.options.debm_simple]
+
+'surface.models' = "debm_simple"
+'surface.debm_simple.albedo_input.file' = "none"
+'surface.debm_simple.albedo_input.periodic' = "yes"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 25
+'surface.debm_simple.c2' = -103
+'surface.debm_simple.refreeze' = 0.50
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 271.65
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.0
+
+[surface.options.debm_enhanced]
+
+'surface.models' = "debm_enhanced"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 29
+'surface.debm_simple.c2' = -93
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 273.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[bed_deformation]
+
+model = "none"
+
+[bed_deformation.options.none]
+
+
+[inverse]
+
+'inverse.file' = "none"
+'inverse.stress_balance.velocity_scale' = 1e2
+'inverse.stress_balance.length_scale' = 1e3
+'inverse.design.cH1' = 1
+'inverse.design.cL2' = 0
+'inverse.max_iterations' = 50
+'inverse.design.param' = "exp"
+'inverse.stress_balance.method' = "tikhonov_blmvm"
+'inverse.stress_balance.tauc_min' = 1e4
+'inverse.stress_balance.tauc_max' = 5e7
+'inverse.tikhonov.penalty_weight' = 10
+'inverse.use_zeta_fixed_mask' = "yes"
+'inverse.adjoint.method' = "exact"
+'inverse.state_func' = "huber"
+'inverse.huber.delta' = 1e2
+
+[solver.forward]
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_pc_mg_galerkin = "both"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
+
+[solver.inverse]
+inv_adj_ksp_type = "gmres"
+inv_adj_pc_type = "jacobi"
+bp_ksp_rtol = 1e-4
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_mg_levels_ksp_type = "chebyshev"
+bp_pc_mg_levels = 3
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_snes_ksp_ew = 0
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-4
diff --git a/pism_terra/config/rgi_era5-mean_frank.toml b/pism_terra/config/rgi_era5-mean_frank.toml
new file mode 100644
index 0000000..2671702
--- /dev/null
+++ b/pism_terra/config/rgi_era5-mean_frank.toml
@@ -0,0 +1,204 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Calibration Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5-mean"
+dem = "glo_90"
+ice_thickness = "frank"
+bathymetry = "gebco"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "all"
+
+[time]
+
+'time.start' = "0001-01-01"
+'time.end' = "0006-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "0001-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "ice_mass,mass_fluxes,effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.Lbz' = 0
+'grid.Lz' = 2000
+'grid.Mbz' = 1
+'grid.Mz' = 101
+'grid.registration' = "center"
+
+[energy]
+
+model = "none"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 3.1689e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 250
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "pdd"
+
+[surface.options.cosipy]
+
+'surface.models' = "given,forcing"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.file' = "none"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple,forcing"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 30
+'surface.debm_simple.c2' = -75
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 272.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+'surface.force_to_thickness.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given,elevation_change"
+'atmosphere.given.file' = "none"
+'atmosphere.elevation_change.temperature_lapse_rate' = 6
+'atmosphere.elevation_change.file' = "none"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.models' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options]
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[solver.forward]
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/rgi_era5-mean_maffezzoli.toml b/pism_terra/config/rgi_era5-mean_maffezzoli.toml
new file mode 100644
index 0000000..141652b
--- /dev/null
+++ b/pism_terra/config/rgi_era5-mean_maffezzoli.toml
@@ -0,0 +1,204 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Calibration Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5-mean"
+dem = "glo_90"
+ice_thickness = "maffezzoli"
+bathymetry = "gebco"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "all"
+
+[time]
+
+'time.start' = "0001-01-01"
+'time.end' = "0006-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "0001-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "ice_mass,mass_fluxes,effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.Lbz' = 0
+'grid.Lz' = 2000
+'grid.Mbz' = 1
+'grid.Mz' = 101
+'grid.registration' = "center"
+
+[energy]
+
+model = "none"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 3.1689e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 250
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "pdd"
+
+[surface.options.cosipy]
+
+'surface.models' = "given,forcing"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.file' = "none"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple,forcing"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 30
+'surface.debm_simple.c2' = -75
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 272.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+'surface.force_to_thickness.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given,elevation_change"
+'atmosphere.given.file' = "none"
+'atmosphere.elevation_change.temperature_lapse_rate' = 6
+'atmosphere.elevation_change.file' = "none"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.models' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options]
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[solver.forward]
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/rgi_era5-mean_millan.toml b/pism_terra/config/rgi_era5-mean_millan.toml
new file mode 100644
index 0000000..8ad841d
--- /dev/null
+++ b/pism_terra/config/rgi_era5-mean_millan.toml
@@ -0,0 +1,204 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Calibration Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5-mean"
+dem = "glo_90"
+ice_thickness = "millan"
+bathymetry = "gebco"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "all"
+
+[time]
+
+'time.start' = "0001-01-01"
+'time.end' = "0006-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "0001-01-01"
+
+[time_stepping]
+
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "ice_mass,mass_fluxes,effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.Lbz' = 0
+'grid.Lz' = 2000
+'grid.Mbz' = 1
+'grid.Mz' = 201
+'grid.registration' = "center"
+
+[energy]
+
+model = "none"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 3.1689e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 250
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "pdd"
+
+[surface.options.cosipy]
+
+'surface.models' = "given,forcing"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.file' = "none"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple,forcing"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 30
+'surface.debm_simple.c2' = -75
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 272.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+'surface.force_to_thickness.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given,elevation_change"
+'atmosphere.given.file' = "none"
+'atmosphere.elevation_change.temperature_lapse_rate' = 6
+'atmosphere.elevation_change.file' = "none"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.models' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options]
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[solver.forward]
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/era5_ec2.toml b/pism_terra/config/rgi_era5_frank.toml
similarity index 72%
rename from pism_terra/config/era5_ec2.toml
rename to pism_terra/config/rgi_era5_frank.toml
index 2a6a7cf..3ad0b03 100644
--- a/pism_terra/config/era5_ec2.toml
+++ b/pism_terra/config/rgi_era5_frank.toml
@@ -1,11 +1,3 @@
-[run]
-
-executable = "pism"
-mpi = "mpirun -np {{ ntasks }}"
-ntasks = 32
-
-[job]
-
[run_info]
'run_info.institution' = "University of Alaska Fairbanks"
@@ -15,12 +7,15 @@ ntasks = 32
bucket = "pism-cloud-data"
climate = "era5"
-dem = "glo_30"
-ice_thickness = "maffezzoli"
+dem = "glo_90"
+ice_thickness = "frank"
+bathymetry = "gebco"
name = "Calibration"
-prefix = "glacier"
-rgi_file = "rgi_c.gpkg"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
velocity = "its_live"
+forcing_mask = "glacier"
[time]
@@ -44,6 +39,7 @@ velocity = "its_live"
['input']
'input.bootstrap' = "yes"
+'input.file' = "none"
'input.forcing.time_extrapolation' = "yes"
'input.forcing.buffer_size' = 390
@@ -95,31 +91,18 @@ model = 'blatter'
[stress_balance.options.blatter]
'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
'stress_balance.blatter.use_eta_transform' = 'yes'
-'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
'stress_balance.sia.surface_gradient_method' = 'eta'
'stress_balance.sia.max_diffusivity' = 100000.0
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
[stress_balance.options.hybrid]
'stress_balance.model' = "ssa+sia"
'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
'stress_balance.sia.surface_gradient_method' = 'eta'
'stress_balance.sia.max_diffusivity' = 100000.0
@@ -133,6 +116,7 @@ bp_ksp_rtol = 1e-3
'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
'stress_balance.blatter.enhancement_factor' = 2.0
'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.bed_smoother.range' = 5e2
[surface]
@@ -145,11 +129,27 @@ model = "pdd"
[surface.options.pdd]
'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.file' = "none"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple,forcing"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 30
+'surface.debm_simple.c2' = -75
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 272.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+'surface.force_to_thickness.file' = "none"
[calving]
'calving.methods' = 'vonmises_calving'
'calving.vonmises_calving.sigma_max' = 7.5e5
+
[atmosphere]
model = "given"
@@ -163,8 +163,18 @@ model = "given"
[ocean]
-'ocean.constant.melt_rate' = 0.0
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.models' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
[hydrology]
@@ -177,3 +187,21 @@ model = "null"
model = "off"
[frontal_melt.options.off]
+
+[solver.forward]
+bp_ksp_monitor = ""
+bp_ksp_view_singularvalues = ""
+bp_snes_monitor_ratio = ""
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/rgi_era5_frank_1year.toml b/pism_terra/config/rgi_era5_frank_1year.toml
new file mode 100644
index 0000000..0d04be1
--- /dev/null
+++ b/pism_terra/config/rgi_era5_frank_1year.toml
@@ -0,0 +1,207 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Calibration Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5"
+dem = "glo_90"
+ice_thickness = "frank"
+bathymetry = "gebco"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "glacier"
+
+[time]
+
+'time.start' = "1978-01-01"
+'time.end' = "1979-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "1978-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 250
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "ice_mass,mass_fluxes,effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.Lbz' = 0
+'grid.Lz' = 2000
+'grid.Mbz' = 1
+'grid.Mz' = 101
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "pdd"
+
+[surface.options.cosipy]
+
+'surface.models' = "given,forcing"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.file' = "none"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple,forcing"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 30
+'surface.debm_simple.c2' = -75
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 272.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+'surface.force_to_thickness.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given,elevation_change"
+'atmosphere.given.file' = "none"
+'atmosphere.elevation_change.temperature_lapse_rate' = 6
+'atmosphere.elevation_change.file' = "none"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.models' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options]
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[solver.forward]
+bp_ksp_monitor = ""
+bp_ksp_view_singularvalues = ""
+bp_snes_monitor_ratio = ""
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/rgi_era5_maffezzoli.toml b/pism_terra/config/rgi_era5_maffezzoli.toml
new file mode 100644
index 0000000..693f478
--- /dev/null
+++ b/pism_terra/config/rgi_era5_maffezzoli.toml
@@ -0,0 +1,208 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Calibration Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5"
+dem = "glo_90"
+ice_thickness = "maffezzoli"
+bathymetry = "gebco"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "glacier"
+heatflux = "lucazeau"
+
+[time]
+
+'time.start' = "1986-01-01"
+'time.end' = "2021-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "1978-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 250
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "ice_mass,mass_fluxes,effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "400m"
+'grid.Lbz' = 0
+'grid.Lz' = 2000
+'grid.Mbz' = 1
+'grid.Mz' = 101
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "pdd"
+
+[surface.options.cosipy]
+
+'surface.models' = "given,forcing"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.file' = "none"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple,forcing"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 30
+'surface.debm_simple.c2' = -75
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 272.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+'surface.force_to_thickness.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given,elevation_change"
+'atmosphere.given.file' = "none"
+'atmosphere.elevation_change.temperature_lapse_rate' = 6
+'atmosphere.elevation_change.file' = "none"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.models' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options]
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[solver.forward]
+bp_ksp_monitor = ""
+bp_ksp_view_singularvalues = ""
+bp_snes_monitor_ratio = ""
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/era5_ec2_1year.toml b/pism_terra/config/rgi_era5_maffezzoli_1year.toml
similarity index 72%
rename from pism_terra/config/era5_ec2_1year.toml
rename to pism_terra/config/rgi_era5_maffezzoli_1year.toml
index 46da0d7..a2f1e22 100644
--- a/pism_terra/config/era5_ec2_1year.toml
+++ b/pism_terra/config/rgi_era5_maffezzoli_1year.toml
@@ -1,11 +1,3 @@
-[run]
-
-executable = "pism"
-mpi = "mpirun -np {{ ntasks }}"
-ntasks = 32
-
-[job]
-
[run_info]
'run_info.institution' = "University of Alaska Fairbanks"
@@ -15,12 +7,15 @@ ntasks = 32
bucket = "pism-cloud-data"
climate = "era5"
-dem = "glo_30"
+dem = "glo_90"
ice_thickness = "maffezzoli"
+bathymetry = "gebco"
name = "Calibration"
-prefix = "glacier"
-rgi_file = "rgi_c.gpkg"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
velocity = "its_live"
+forcing_mask = "glacier"
[time]
@@ -44,6 +39,7 @@ velocity = "its_live"
['input']
'input.bootstrap' = "yes"
+'input.file' = "none"
'input.forcing.time_extrapolation' = "yes"
'input.forcing.buffer_size' = 390
@@ -62,7 +58,7 @@ velocity = "its_live"
[grid]
-resolution = "400m"
+resolution = "500m"
'grid.Lbz' = 0
'grid.Lz' = 2000
'grid.Mbz' = 1
@@ -95,31 +91,18 @@ model = 'blatter'
[stress_balance.options.blatter]
'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
'stress_balance.blatter.use_eta_transform' = 'yes'
-'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
'stress_balance.sia.surface_gradient_method' = 'eta'
'stress_balance.sia.max_diffusivity' = 100000.0
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
[stress_balance.options.hybrid]
'stress_balance.model' = "ssa+sia"
'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
'stress_balance.sia.surface_gradient_method' = 'eta'
'stress_balance.sia.max_diffusivity' = 100000.0
@@ -133,6 +116,7 @@ bp_ksp_rtol = 1e-3
'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
'stress_balance.blatter.enhancement_factor' = 2.0
'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.bed_smoother.range' = 5e2
[surface]
@@ -145,11 +129,27 @@ model = "pdd"
[surface.options.pdd]
'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.file' = "none"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple,forcing"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 30
+'surface.debm_simple.c2' = -75
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 272.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+'surface.force_to_thickness.file' = "none"
[calving]
'calving.methods' = 'vonmises_calving'
'calving.vonmises_calving.sigma_max' = 7.5e5
+
[atmosphere]
model = "given"
@@ -163,8 +163,18 @@ model = "given"
[ocean]
-'ocean.constant.melt_rate' = 0.0
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.models' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
[hydrology]
@@ -177,3 +187,21 @@ model = "null"
model = "off"
[frontal_melt.options.off]
+
+[solver.forward]
+bp_ksp_monitor = ""
+bp_ksp_view_singularvalues = ""
+bp_snes_monitor_ratio = ""
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/test_era5_local.toml b/pism_terra/config/rgi_era5_mean_2years.toml
similarity index 69%
rename from pism_terra/config/test_era5_local.toml
rename to pism_terra/config/rgi_era5_mean_2years.toml
index e38f8f7..17ff554 100644
--- a/pism_terra/config/test_era5_local.toml
+++ b/pism_terra/config/rgi_era5_mean_2years.toml
@@ -1,15 +1,3 @@
-[run]
-
-executable = "pism"
-mpi = ""
-ntasks = 8
-
-[job]
-
-queue = "t2small"
-walltime = "48:00:00"
-nodes = 1
-
[run_info]
'run_info.institution' = "University of Alaska Fairbanks"
@@ -18,24 +6,27 @@ nodes = 1
[campaign]
bucket = "pism-cloud-data"
-climate = "era5"
-dem = "glo_30"
+climate = "era5-monthly-mean"
+dem = "glo_90"
ice_thickness = "maffezzoli"
+bathymetry = "gebco"
name = "Calibration"
-prefix = "glacier"
-rgi_file = "rgi_c.gpkg"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
velocity = "its_live"
+forcing_mask = "glacier"
[time]
-'time.start' = "1978-01-01"
-'time.end' = "1978-07-01"
-'time.calendar' = "standard"
-'time.reference_date' = "1978-01-01"
+'time.start' = "0001-01-01"
+'time.end' = "0003-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "0001-01-01"
[time_stepping]
-'time_stepping.adaptive_ratio' = 250
+'time_stepping.adaptive_ratio' = 500
'time_stepping.skip.enabled' = "yes"
'time_stepping.skip.max' = 100
@@ -48,26 +39,27 @@ velocity = "its_live"
['input']
'input.bootstrap' = "yes"
+'input.file' = "none"
'input.forcing.time_extrapolation' = "yes"
'input.forcing.buffer_size' = 390
[reporting]
-'output.format' = "netcdf4_serial"
+'output.format' = "netcdf4_parallel"
'output.compression_level'= 2
'output.size' = "medium"
'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
'output.spatial.file' = "none"
'output.spatial.times' = "monthly"
-'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf"
-'output.scalar.times' = "daily"
-'output.scalar.file' = "none"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes"
'output.checkpoint.interval' = 24
-'output.extra.stop_missing' = 'no'
+'output.spatial.stop_missing' = 'no'
[grid]
-resolution = "1000m"
+resolution = "500m"
+'grid.file' = "none"
'grid.Lbz' = 0
'grid.Lz' = 2000
'grid.Mbz' = 1
@@ -100,31 +92,18 @@ model = 'blatter'
[stress_balance.options.blatter]
'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
'stress_balance.blatter.use_eta_transform' = 'yes'
-'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
'stress_balance.sia.surface_gradient_method' = 'eta'
'stress_balance.sia.max_diffusivity' = 100000.0
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
[stress_balance.options.hybrid]
'stress_balance.model' = "ssa+sia"
'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
'stress_balance.sia.surface_gradient_method' = 'eta'
'stress_balance.sia.max_diffusivity' = 100000.0
@@ -137,7 +116,10 @@ bp_ksp_rtol = 1e-3
'basal_yield_stress.model' = 'mohr_coulomb'
'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
[surface]
@@ -150,26 +132,39 @@ model = "pdd"
[surface.options.pdd]
'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.alpha' = 0.95
+'surface.force_to_thickness.ice_free_alpha_factor' = 10.0
+'surface.force_to_thickness.file' = "none"
[calving]
'calving.methods' = 'vonmises_calving'
'calving.vonmises_calving.sigma_max' = 7.5e5
+
[atmosphere]
model = "given"
[atmosphere.options.given]
-'atmosphere.models' = "given,elevation_change"
+'atmosphere.models' = "given"
'atmosphere.given.file' = "none"
-'atmosphere.elevation_change.temperature_lapse_rate' = 6
-'atmosphere.elevation_change.file' = "none"
+'atmosphere.given.periodic' = "yes"
[ocean]
-'ocean.constant.melt_rate' = 0.0
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
[hydrology]
@@ -182,3 +177,21 @@ model = "null"
model = "off"
[frontal_melt.options.off]
+
+[solver.forward]
+bp_ksp_monitor = ""
+bp_ksp_view_singularvalues = ""
+bp_snes_monitor_ratio = ""
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/rgi_era5_millan.toml b/pism_terra/config/rgi_era5_millan.toml
new file mode 100644
index 0000000..81312d8
--- /dev/null
+++ b/pism_terra/config/rgi_era5_millan.toml
@@ -0,0 +1,206 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Calibration Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5"
+dem = "glo_90"
+ice_thickness = "millan"
+bathymetry = "gebco"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "glacier"
+
+[time]
+
+'time.start' = "1978-01-01"
+'time.end' = "2025-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "1978-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 250
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "ice_mass,mass_fluxes,effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "400m"
+'grid.Lbz' = 0
+'grid.Lz' = 2000
+'grid.Mbz' = 1
+'grid.Mz' = 101
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "pdd"
+
+[surface.options.cosipy]
+
+'surface.models' = "given,forcing"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.file' = "none"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple,forcing"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 30
+'surface.debm_simple.c2' = -75
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 272.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+'surface.force_to_thickness.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given,elevation_change"
+'atmosphere.given.file' = "none"
+'atmosphere.elevation_change.temperature_lapse_rate' = 6
+'atmosphere.elevation_change.file' = "none"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.models' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options]
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[solver.forward]
+bp_ksp_monitor = ""
+bp_snes_monitor_ratio = ""
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/rgi_forward_frank.toml b/pism_terra/config/rgi_forward_frank.toml
new file mode 100644
index 0000000..47ca149
--- /dev/null
+++ b/pism_terra/config/rgi_forward_frank.toml
@@ -0,0 +1,242 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Initialization Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5"
+dem = "glo_90"
+ice_thickness = "frank"
+bathymetry = "gebco"
+heatflux = "lucazeau"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "none"
+
+[time]
+
+'time.start' = "1980-01-01"
+'time.end' = "2025-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "1980-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 500
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes,mass_fluxes"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.file' = "none"
+'grid.Lbz' = 0
+'grid.Lz' = 1200
+'grid.Mbz' = 1
+'grid.Mz' = 61
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 0.12
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 35
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "debm"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd"
+'surface.pdd.std_dev.file' = "none"
+'surface.pdd.std_dev.periodic' = "yes"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 25
+'surface.debm_simple.c2' = -103
+'surface.debm_simple.refreeze' = 0.50
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 271.65
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.0
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[inverse]
+
+'inverse.file' = "none"
+'inverse.stress_balance.velocity_scale' = 1e2
+'inverse.stress_balance.length_scale' = 1e3
+'inverse.design.cH1' = 1
+'inverse.design.cL2' = 0
+'inverse.max_iterations' = 50
+'inverse.design.param' = "exp"
+'inverse.stress_balance.method' = "tikhonov_blmvm"
+'inverse.stress_balance.tauc_min' = 1e4
+'inverse.stress_balance.tauc_max' = 5e7
+'inverse.tikhonov.penalty_weight' = 10
+'inverse.use_zeta_fixed_mask' = "yes"
+'inverse.adjoint.method' = "exact"
+'inverse.state_func' = "huber"
+'inverse.huber.delta' = 1e2
+
+[solver.forward]
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
+
+[solver.inverse]
+inv_adj_ksp_type = "gmres"
+inv_adj_pc_type = "jacobi"
+bp_ksp_rtol = 1e-4
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_mg_levels_ksp_type = "chebyshev"
+bp_pc_mg_levels = 3
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_snes_ksp_ew = 0
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-4
diff --git a/pism_terra/config/rgi_forward_maffezzoli.toml b/pism_terra/config/rgi_forward_maffezzoli.toml
new file mode 100644
index 0000000..a4e0168
--- /dev/null
+++ b/pism_terra/config/rgi_forward_maffezzoli.toml
@@ -0,0 +1,258 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Initialization Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5"
+dem = "glo_90"
+ice_thickness = "maffezzoli"
+bathymetry = "gebco"
+heatflux = "lucazeau"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "none"
+
+[time]
+
+'time.start' = "1986-01-01"
+'time.end' = "2026-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "1980-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 500
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes,mass_fluxes"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.file' = "none"
+'grid.Lbz' = 0
+'grid.Lz' = 1200
+'grid.Mbz' = 1
+'grid.Mz' = 61
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 0.12
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 35
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "debm_simple"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd"
+'surface.pdd.std_dev.file' = "none"
+'surface.pdd.std_dev.periodic' = "yes"
+
+[surface.options.debm_simple]
+
+'surface.models' = "debm_simple"
+'surface.debm_simple.albedo_input.file' = "none"
+'surface.debm_simple.albedo_input.periodic' = "yes"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 25
+'surface.debm_simple.c2' = -103
+'surface.debm_simple.refreeze' = 0.50
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 271.65
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.0
+
+[surface.options.debm_enhanced]
+
+'surface.models' = "debm_enhanced"
+'surface.debm_simple.albedo_input.file' = "none"
+'surface.debm_simple.albedo_input.periodic' = "yes"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 25
+'surface.debm_simple.c2' = -103
+'surface.debm_simple.refreeze' = 0.50
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 271.65
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.0
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[inverse]
+
+'inverse.file' = "none"
+'inverse.stress_balance.velocity_scale' = 1e2
+'inverse.stress_balance.length_scale' = 1e3
+'inverse.design.cH1' = 1
+'inverse.design.cL2' = 0
+'inverse.max_iterations' = 50
+'inverse.design.param' = "exp"
+'inverse.stress_balance.method' = "tikhonov_blmvm"
+'inverse.stress_balance.tauc_min' = 1e4
+'inverse.stress_balance.tauc_max' = 5e7
+'inverse.tikhonov.penalty_weight' = 10
+'inverse.use_zeta_fixed_mask' = "yes"
+'inverse.adjoint.method' = "exact"
+'inverse.state_func' = "huber"
+'inverse.huber.delta' = 1e2
+
+[solver.forward]
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
+
+[solver.inverse]
+inv_adj_ksp_type = "gmres"
+inv_adj_pc_type = "jacobi"
+bp_ksp_rtol = 1e-4
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_mg_levels_ksp_type = "chebyshev"
+bp_pc_mg_levels = 3
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_snes_ksp_ew = 0
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-4
diff --git a/pism_terra/config/rgi_forward_millan.toml b/pism_terra/config/rgi_forward_millan.toml
new file mode 100644
index 0000000..6b404d3
--- /dev/null
+++ b/pism_terra/config/rgi_forward_millan.toml
@@ -0,0 +1,242 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Initialization Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5"
+dem = "glo_90"
+ice_thickness = "millan"
+bathymetry = "gebco"
+heatflux = "lucazeau"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "none"
+
+[time]
+
+'time.start' = "1980-01-01"
+'time.end' = "2025-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "1980-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 500
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "monthly"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes,mass_fluxes"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.file' = "none"
+'grid.Lbz' = 0
+'grid.Lz' = 1200
+'grid.Mbz' = 1
+'grid.Mz' = 61
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 0.12
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 35
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "debm"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd"
+'surface.pdd.std_dev.file' = "none"
+'surface.pdd.std_dev.periodic' = "yes"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 25
+'surface.debm_simple.c2' = -103
+'surface.debm_simple.refreeze' = 0.50
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 271.65
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.0
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[inverse]
+
+'inverse.file' = "none"
+'inverse.stress_balance.velocity_scale' = 1e2
+'inverse.stress_balance.length_scale' = 1e3
+'inverse.design.cH1' = 1
+'inverse.design.cL2' = 0
+'inverse.max_iterations' = 50
+'inverse.design.param' = "exp"
+'inverse.stress_balance.method' = "tikhonov_blmvm"
+'inverse.stress_balance.tauc_min' = 1e4
+'inverse.stress_balance.tauc_max' = 5e7
+'inverse.tikhonov.penalty_weight' = 10
+'inverse.use_zeta_fixed_mask' = "yes"
+'inverse.adjoint.method' = "exact"
+'inverse.state_func' = "huber"
+'inverse.huber.delta' = 1e2
+
+[solver.forward]
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
+
+[solver.inverse]
+inv_adj_ksp_type = "gmres"
+inv_adj_pc_type = "jacobi"
+bp_ksp_rtol = 1e-4
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_mg_levels_ksp_type = "chebyshev"
+bp_pc_mg_levels = 3
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_snes_ksp_ew = 0
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-4
diff --git a/pism_terra/config/rgi_init_maffezzoli.toml b/pism_terra/config/rgi_init_maffezzoli.toml
new file mode 100644
index 0000000..4048ca7
--- /dev/null
+++ b/pism_terra/config/rgi_init_maffezzoli.toml
@@ -0,0 +1,263 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Initialization Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5-mean"
+dem = "glo_90"
+ice_thickness = "maffezzoli"
+bathymetry = "gebco"
+heatflux = "lucazeau"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "all"
+
+[time]
+
+'time.start' = "0001-01-01"
+'time.end' = "0006-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "0001-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 500
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+'geometry.update.enabled' = "no"
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = 50
+'output.spatial.vars' = "velsurf_mag,velbase_mag,mask"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.file' = "none"
+'grid.Lbz' = 0
+'grid.Lz' = 1200
+'grid.Mbz' = 1
+'grid.Mz' = 61
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'energy.bedrock_thermal.file' = "none"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.none]
+
+'stress_balance.model' = "none"
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 0.12
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 35
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "pdd"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd"
+'surface.pdd.std_dev.file' = "none"
+'surface.pdd.std_dev.periodic' = "yes"
+
+[surface.options.debm_simple]
+
+'surface.models' = "debm_simple"
+'surface.debm_simple.albedo_input.file' = "none"
+'surface.debm_simple.albedo_input.periodic' = "yes"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 25
+'surface.debm_simple.c2' = -103
+'surface.debm_simple.refreeze' = 0.50
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 271.65
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.0
+
+[surface.options.debm_enhanced]
+
+'surface.models' = "debm_enhanced"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 29
+'surface.debm_simple.c2' = -93
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 273.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[inverse]
+
+'inverse.file' = "none"
+'inverse.stress_balance.velocity_scale' = 1e2
+'inverse.stress_balance.length_scale' = 1e3
+'inverse.design.cH1' = 1
+'inverse.design.cL2' = 0
+'inverse.max_iterations' = 50
+'inverse.design.param' = "exp"
+'inverse.stress_balance.method' = "tikhonov_blmvm"
+'inverse.stress_balance.tauc_min' = 1e4
+'inverse.stress_balance.tauc_max' = 5e7
+'inverse.tikhonov.penalty_weight' = 10
+'inverse.use_zeta_fixed_mask' = "yes"
+'inverse.adjoint.method' = "exact"
+'inverse.state_func' = "huber"
+'inverse.huber.delta' = 1e2
+
+[solver]
+
+[solver.forward]
+
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type ="sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
+
+[solver.inverse]
+
+'bp_ksp_rtol' = 1e-4
+'bp_mg_coarse_ksp_type' = "gmres"
+'bp_mg_coarse_pc_type' = "gamg"
+'bp_mg_coarse_ksp_rtol' = 1e-2
+'bp_mg_coarse_ksp_max_it' = 50
+'bp_mg_levels_ksp_type' = "chebyshev"
+'bp_pc_mg_levels' = 3
+'bp_ksp_type' = "fgmres"
+'bp_pc_type' = "mg"
+'bp_snes_ksp_ew' = 0
+'bp_snes_ksp_ew_version' = 3
+'bp_snes_rtol' = 1e-4
+inv_adj_ksp_type = "gmres"
+inv_adj_pc_type = "jacobi"
diff --git a/pism_terra/config/rgi_inverse_gpbld_frank.toml b/pism_terra/config/rgi_inverse_gpbld_frank.toml
new file mode 100644
index 0000000..9d756fd
--- /dev/null
+++ b/pism_terra/config/rgi_inverse_gpbld_frank.toml
@@ -0,0 +1,239 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Initialization Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5-monthly-mean"
+dem = "glo_90"
+ice_thickness = "frank"
+bathymetry = "gebco"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "all"
+heatflux = "lucazeau"
+
+[time]
+
+'time.start' = "0001-01-01"
+'time.end' = "0006-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "0001-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 500
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.file' = "none"
+'grid.Lbz' = 0
+'grid.Lz' = 1200
+'grid.Mbz' = 1
+'grid.Mz' = 61
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 0.12
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 35
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "pdd"
+
+[surface.options.cosipy]
+
+'surface.models' = "given,forcing"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.alpha' = 0.75
+'surface.force_to_thickness.ice_free_alpha_factor' = 10.0
+'surface.force_to_thickness.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[bed_deformation]
+
+model = "none"
+
+[bed_deformation.options.none]
+
+[inverse]
+
+'inverse.file' = "none"
+'inverse.stress_balance.velocity_scale' = 1e2
+'inverse.stress_balance.length_scale' = 1e3
+'inverse.design.cH1' = 1
+'inverse.design.cL2' = 0
+'inverse.max_iterations' = 50
+'inverse.design.param' = "exp"
+'inverse.stress_balance.method' = "tikhonov_blmvm"
+'inverse.stress_balance.tauc_min' = 1e4
+'inverse.stress_balance.tauc_max' = 5e7
+'inverse.tikhonov.penalty_weight' = 10
+'inverse.use_zeta_fixed_mask' = "yes"
+'inverse.adjoint.method' = "exact"
+'inverse.state_func' = "huber"
+'inverse.huber.delta' = 1e2
+
+[solver.forward]
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
+
+[solver.inverse]
+inv_adj_ksp_type = "gmres"
+inv_adj_pc_type = "jacobi"
+bp_ksp_rtol = 1e-4
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_mg_levels_ksp_type = "chebyshev"
+bp_pc_mg_levels = 3
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_snes_ksp_ew = 0
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-4
diff --git a/pism_terra/config/rgi_inverse_gpbld_maffezzoli.toml b/pism_terra/config/rgi_inverse_gpbld_maffezzoli.toml
new file mode 100644
index 0000000..0d3a7bc
--- /dev/null
+++ b/pism_terra/config/rgi_inverse_gpbld_maffezzoli.toml
@@ -0,0 +1,242 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Initialization Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5-monthly-mean"
+dem = "glo_90"
+ice_thickness = "maffezzoli"
+bathymetry = "gebco"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "all"
+heatflux = "lucazeau"
+
+[time]
+
+'time.start' = "0001-01-01"
+'time.end' = "0006-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "0001-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 1000
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 24
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.file' = "none"
+'grid.Lbz' = 0
+'grid.Lz' = 1200
+'grid.Mbz' = 1
+'grid.Mz' = 61
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 0.12
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 35
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "pdd"
+
+[surface.options.cosipy]
+
+'surface.models' = "given,forcing"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.alpha' = 0.95
+'surface.force_to_thickness.ice_free_alpha_factor' = 10.0
+'surface.force_to_thickness.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[bed_deformation]
+
+model = "none"
+
+[bed_deformation.options.none]
+
+[inverse]
+
+'inverse.file' = "none"
+'inverse.stress_balance.velocity_scale' = 1e2
+'inverse.stress_balance.length_scale' = 1e3
+'inverse.design.cH1' = 1
+'inverse.design.cL2' = 0
+'inverse.max_iterations' = 250
+'inverse.design.param' = "exp"
+'inverse.stress_balance.method' = "tikhonov_blmvm"
+'inverse.stress_balance.tauc_min' = 1e4
+'inverse.stress_balance.tauc_max' = 5e7
+'inverse.tikhonov.penalty_weight' = 10
+'inverse.use_zeta_fixed_mask' = "yes"
+'inverse.adjoint.method' = "incomplete"
+'inverse.state_func' = "huber"
+'inverse.huber.delta' = 1e2
+'inverse.use_design_prior' = "yes"
+
+[solver.forward]
+
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
+
+[solver.inverse]
+
+inv_adj_ksp_type = "gmres"
+inv_adj_pc_type = "jacobi"
+bp_ksp_rtol = 1e-4
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_mg_levels_ksp_type = "chebyshev"
+bp_pc_mg_levels = 3
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_snes_ksp_ew = 0
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-4
diff --git a/pism_terra/config/rgi_inverse_gpbld_millan.toml b/pism_terra/config/rgi_inverse_gpbld_millan.toml
new file mode 100644
index 0000000..a5830db
--- /dev/null
+++ b/pism_terra/config/rgi_inverse_gpbld_millan.toml
@@ -0,0 +1,239 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Initialization Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "era5-monthly-mean"
+dem = "glo_90"
+ice_thickness = "millan"
+bathymetry = "gebco"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "all"
+heatflux = "lucazeau"
+
+[time]
+
+'time.start' = "0001-01-01"
+'time.end' = "0006-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "0001-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 500
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.file' = "none"
+'grid.Lbz' = 0
+'grid.Lz' = 1200
+'grid.Mbz' = 1
+'grid.Mz' = 61
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 0.12
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 35
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "pdd"
+
+[surface.options.cosipy]
+
+'surface.models' = "given,forcing"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.alpha' = 0.75
+'surface.force_to_thickness.ice_free_alpha_factor' = 10.0
+'surface.force_to_thickness.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[bed_deformation]
+
+model = "none"
+
+[bed_deformation.options.none]
+
+[inverse]
+
+'inverse.file' = "none"
+'inverse.stress_balance.velocity_scale' = 1e2
+'inverse.stress_balance.length_scale' = 1e3
+'inverse.design.cH1' = 1
+'inverse.design.cL2' = 0
+'inverse.max_iterations' = 50
+'inverse.design.param' = "exp"
+'inverse.stress_balance.method' = "tikhonov_blmvm"
+'inverse.stress_balance.tauc_min' = 1e4
+'inverse.stress_balance.tauc_max' = 5e7
+'inverse.tikhonov.penalty_weight' = 10
+'inverse.use_zeta_fixed_mask' = "yes"
+'inverse.adjoint.method' = "exact"
+'inverse.state_func' = "huber"
+'inverse.huber.delta' = 1e2
+
+[solver.forward]
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
+
+[solver.inverse]
+inv_adj_ksp_type = "gmres"
+inv_adj_pc_type = "jacobi"
+bp_ksp_rtol = 1e-4
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_mg_levels_ksp_type = "chebyshev"
+bp_pc_mg_levels = 3
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_snes_ksp_ew = 0
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-4
diff --git a/pism_terra/config/rgi_snap-mean_forward_maffezzoli.toml b/pism_terra/config/rgi_snap-mean_forward_maffezzoli.toml
new file mode 100644
index 0000000..5e72691
--- /dev/null
+++ b/pism_terra/config/rgi_snap-mean_forward_maffezzoli.toml
@@ -0,0 +1,257 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Initialization Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "snap-monthly-mean"
+dem = "glo_90"
+ice_thickness = "maffezzoli"
+bathymetry = "gebco"
+name = "Calibration"
+prefix = "rgi/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "none"
+heatflux = "lucazeau"
+
+[time]
+
+'time.start' = "0001-01-01"
+'time.end' = "0251-01-01"
+'time.calendar' = "365_day"
+'time.reference_date' = "0001-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 500
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,pdd_fluxes,ice_mass,mass_fluxes"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "500m"
+'grid.file' = "none"
+'grid.Lbz' = 0
+'grid.Lz' = 1200
+'grid.Mbz' = 1
+'grid.Mz' = 61
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+'time_stepping.adaptive_ratio' = 0.12
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 35
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "debm_enhanced"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd"
+
+[surface.options.debm_simple]
+
+'surface.models' = "debm_simple"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 29
+'surface.debm_simple.c2' = -93
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 273.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+
+[surface.options.debm_enhanced]
+
+'surface.models' = "debm_enhanced"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 29
+'surface.debm_simple.c2' = -93
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 273.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given"
+'atmosphere.given.file' = "none"
+'atmosphere.given.periodic' = "yes"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options.null]
+
+'hydrology.model' = "null"
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[bed_deformation]
+
+model = "none"
+
+[bed_deformation.options.none]
+
+[inverse]
+
+'inverse.file' = "none"
+'inverse.stress_balance.velocity_scale' = 1e2
+'inverse.stress_balance.length_scale' = 1e3
+'inverse.design.cH1' = 1
+'inverse.design.cL2' = 0
+'inverse.max_iterations' = 50
+'inverse.design.param' = "exp"
+'inverse.stress_balance.method' = "tikhonov_blmvm"
+'inverse.stress_balance.tauc_min' = 1e4
+'inverse.stress_balance.tauc_max' = 5e7
+'inverse.tikhonov.penalty_weight' = 10
+'inverse.use_zeta_fixed_mask' = "yes"
+'inverse.adjoint.method' = "exact"
+'inverse.state_func' = "huber"
+'inverse.huber.delta' = 1e2
+
+[solver.forward]
+
+bp_ksp_rtol = 1e-3
+bp_ksp_type = "fgmres"
+bp_mg_coarse_ksp_max_it = 50
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_pc_mg_levels = 3
+bp_pc_type = "mg"
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+
+[solver.inverse]
+
+inv_adj_ksp_type = "gmres"
+inv_adj_pc_type = "jacobi"
+bp_ksp_rtol = 1e-4
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_mg_levels_ksp_type = "chebyshev"
+bp_pc_mg_levels = 3
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_snes_ksp_ew = 0
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-4
diff --git a/pism_terra/config/s4f_carra2.toml b/pism_terra/config/s4f_carra2.toml
new file mode 100644
index 0000000..cbd70d0
--- /dev/null
+++ b/pism_terra/config/s4f_carra2.toml
@@ -0,0 +1,212 @@
+[run_info]
+
+'run_info.institution' = "University of Alaska Fairbanks"
+'run_info.title' = "PISM-TERRA Calibration Run"
+
+[campaign]
+
+bucket = "pism-cloud-data"
+climate = "carra2"
+dem = "glo_90"
+ice_thickness = "maffezzoli"
+bathymetry = "gebco"
+name = "Hindcast"
+prefix = "s4f/glacier"
+rgi_complex_file = "rgi_c.gpkg"
+rgi_glacier_file = "rgi_g.gpkg"
+velocity = "its_live"
+forcing_mask = "glacier"
+
+[time]
+
+'time.start' = "1980-01-01"
+'time.end' = "2020-01-01"
+'time.calendar' = "standard"
+'time.reference_date' = "1978-01-01"
+
+[time_stepping]
+
+'time_stepping.adaptive_ratio' = 500
+'time_stepping.skip.enabled' = "yes"
+'time_stepping.skip.max' = 100
+
+['geometry']
+
+'geometry.front_retreat.use_cfl' = 'yes'
+'geometry.part_grid.enabled' = 'yes'
+'geometry.remove_icebergs' = 'yes'
+
+['input']
+
+'input.bootstrap' = "yes"
+'input.file' = "none"
+'input.forcing.time_extrapolation' = "yes"
+'input.forcing.buffer_size' = 390
+
+[reporting]
+
+'output.format' = "netcdf4_parallel"
+'output.compression_level'= 2
+'output.size' = "medium"
+'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
+'output.scalar.times' = "daily"
+'output.spatial.file' = "none"
+'output.spatial.times' = "yearly"
+'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf"
+'output.checkpoint.interval' = 24
+'output.spatial.stop_missing' = 'no'
+
+[grid]
+
+resolution = "200m"
+'grid.file' = "none"
+'grid.Lbz' = 0
+'grid.Lz' = 2000
+'grid.Mbz' = 1
+'grid.Mz' = 101
+'grid.registration' = "center"
+
+[energy]
+
+model = "enthalpy"
+
+[energy.options.none]
+
+'energy.model' = "none"
+'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
+'stress_balance.blatter.flow_law' = "isothermal_glen"
+'stress_balance.sia.flow_law' = "isothermal_glen"
+'stress_balance.ssa.flow_law' = "isothermal_glen"
+
+[energy.options.enthalpy]
+
+'energy.model' = "enthalpy"
+'stress_balance.blatter.flow_law' = "gpbld"
+'stress_balance.sia.flow_law' = "gpbld"
+'stress_balance.ssa.flow_law' = "gpbld"
+
+[stress_balance]
+
+model = 'blatter'
+
+[stress_balance.options.blatter]
+
+'stress_balance.model' = "blatter"
+'stress_balance.blatter.Mz' = 10
+'stress_balance.blatter.coarsening_factor' = 3
+'stress_balance.blatter.use_eta_transform' = 'yes'
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[stress_balance.options.hybrid]
+
+'stress_balance.model' = "ssa+sia"
+'stress_balance.ssa.method' = "fd"
+'stress_balance.calving_front_stress_bc' = 'no'
+'stress_balance.sia.surface_gradient_method' = 'eta'
+'stress_balance.sia.max_diffusivity' = 100000.0
+
+[iceflow]
+
+'basal_resistance.pseudo_plastic.q' = 0.75
+'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
+'basal_resistance.pseudo_plastic.enabled' = 'yes'
+'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
+'basal_yield_stress.model' = 'mohr_coulomb'
+'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
+'stress_balance.blatter.enhancement_factor' = 2.0
+'stress_balance.blatter.Glen_exponent' = 3.0
+'stress_balance.sia.enhancement_factor' = 2.0
+'stress_balance.sia.Glen_exponent' = 3.0
+'stress_balance.sia.bed_smoother.range' = 5e2
+
+[surface]
+
+model = "debm"
+
+[surface.options.cosipy]
+
+'surface.models' = "given,forcing"
+
+[surface.options.pdd]
+
+'surface.models' = "pdd,forcing"
+'surface.force_to_thickness.file' = "none"
+
+[surface.options.debm]
+
+'surface.models' = "debm_simple,forcing"
+'surface.debm_simple.std_dev.file' = "none"
+'surface.debm_simple.std_dev.periodic' = "yes"
+'surface.debm_simple.interpret_precip_as_snow' = "no"
+'surface.debm_simple.c1' = 30
+'surface.debm_simple.c2' = -75
+'surface.debm_simple.refreeze' = 0.60
+'surface.debm_simple.refreeze_ice_melt' = "no"
+'surface.debm_simple.air_temp_all_precip_as_snow' = 272.15
+'surface.debm_simple.air_temp_all_precip_as_rain' = 275.15
+'surface.force_to_thickness.file' = "none"
+
+[calving]
+
+'calving.methods' = 'vonmises_calving'
+'calving.vonmises_calving.sigma_max' = 7.5e5
+
+[atmosphere]
+
+model = "given"
+
+[atmosphere.options.given]
+
+'atmosphere.models' = "given,elevation_change"
+'atmosphere.given.file' = "none"
+'atmosphere.elevation_change.temperature_lapse_rate' = 6
+'atmosphere.elevation_change.file' = "none"
+
+[ocean]
+
+model = "constant"
+
+[ocean.options.given]
+
+'ocean.model' = "given"
+'ocean.given.file' = "none"
+'ocean.given.periodic' = "yes"
+
+[ocean.options.constant]
+
+'ocean.models' = 'constant'
+'ocean.constant.melt_rate' = 0.0
+
+[hydrology]
+
+model = "null"
+
+[hydrology.options.null]
+
+'hydrology.null_diffuse_till_water' = "no"
+
+[frontal_melt]
+
+model = "off"
+
+[frontal_melt.options.off]
+
+[solver.forward]
+bp_ksp_monitor = ""
+bp_ksp_view_singularvalues = ""
+bp_snes_monitor_ratio = ""
+bp_ksp_type = "fgmres"
+bp_pc_type = "mg"
+bp_mg_levels_ksp_type = "richardson"
+bp_mg_levels_pc_type = "sor"
+bp_mg_coarse_ksp_type = "gmres"
+bp_mg_coarse_pc_type = "gamg"
+bp_mg_coarse_ksp_rtol = 1e-2
+bp_mg_coarse_ksp_max_it = 50
+bp_pc_mg_levels = 3
+bp_snes_ksp_ew = 1
+bp_snes_ksp_ew_version = 3
+bp_snes_rtol = 1e-3
+bp_ksp_rtol = 1e-3
diff --git a/pism_terra/config/bm_greenland.toml b/pism_terra/config/setup_bedmachine_greenland.toml
similarity index 73%
rename from pism_terra/config/bm_greenland.toml
rename to pism_terra/config/setup_bedmachine_greenland.toml
index 6570667..a6bb3c6 100644
--- a/pism_terra/config/bm_greenland.toml
+++ b/pism_terra/config/setup_bedmachine_greenland.toml
@@ -1,16 +1,17 @@
ismip7_to_pism = {"acabf" = "climatic_mass_balance", "runoff" = "water_input_rate", "tas" = "ice_surface_temp", "TF" = "theta_ocean", "geothermal_heat_flux1" = "bheatflx"}
gcms = ["CESM2-WACCM"]
+prefix = "input"
+version = "v1"
[domain]
x_bounds = [-653000.0, 879700.0]
y_bounds = [-3384350.0, -632750.0]
-resolution = "600m"
+resolution = "900m"
[pathway]
-historical = {"start_year" = 1975, "end_year" = 2015, version = 1}
-ssp585 = {"start_year" = 2015, "end_year" = 2299, version = 1 }
+ssp585 = {"historical" = [2014, 2015], "projection" = [2015, 2016], version = 1 }
[forcing]
diff --git a/pism_terra/config/setup_glaciermip4.toml b/pism_terra/config/setup_glaciermip4.toml
new file mode 100644
index 0000000..3ebde69
--- /dev/null
+++ b/pism_terra/config/setup_glaciermip4.toml
@@ -0,0 +1,5 @@
+[regions]
+
+1 = {name = "alaska", crs = "EPSG:5936"}
+3 = {name = "arctic_canada_north", crs = "EPSG:3413"}
+7 = {name = "svalbard_jan_mayen", crs = "EPSG:3413"}
diff --git a/pism_terra/config/setup_ismip7_greenland.toml b/pism_terra/config/setup_ismip7_greenland.toml
new file mode 100644
index 0000000..c5bda63
--- /dev/null
+++ b/pism_terra/config/setup_ismip7_greenland.toml
@@ -0,0 +1,22 @@
+ismip7_to_pism = {"acabf" = "climatic_mass_balance", "dacabfdz" = "climatic_mass_balance_gradient", "mrro" = "runoff_rate", "dmrrodz" = "runoff_rate_gradient", "ts" = "ice_surface_temp", "dtsdz" = "ice_surface_temp_gradient", "tf" = "theta_ocean", "so" = "salinity_ocean", "geothermal_heat_flux1" = "bheatflx"}
+version = "v2"
+bucket = "pism-cloud-data"
+prefix = "ismip7/greenland/input"
+ice_sheet = "GrIS"
+
+[domain]
+
+crs = "EPSG:3413"
+x_bounds = [-750650.0, 977350.0]
+y_bounds = [-3412550.0, -604550.0]
+resolution = "900m"
+
+[gcms]
+
+'CESM2-WACCM' = {historical = {start = 1978, end = 2014, version = 2}, ssp126 = {start = 2015, end = 2300, version = 2 }, ssp370 = {start = 2015, end = 2100, version = 2 }, ssp585 = {start = 2015, end = 2300, version = 2 } }
+'MRI-ESM2-0' = {historical = {start = 1978, end = 2014, version = 1}, ssp126 = {start = 2015, end = 2300, version = 1 }, ssp370 = {start = 2015, end = 2100, version = 1 }, ssp585 = {start = 2015, end = 2300, version = 1 } }
+
+[forcing]
+
+climate = {fields = ["acabf", "dacabfdz", "mrro", "ts", "dmrrodz", "dtsdz"], short_hand = "SDBN1-1000m"}
+ocean = {fields = ["tf", "so"], short_hand = "ocean"}
diff --git a/pism_terra/config/setup_kitp_greenland.toml b/pism_terra/config/setup_kitp_greenland.toml
index c55a4bf..c37c12f 100644
--- a/pism_terra/config/setup_kitp_greenland.toml
+++ b/pism_terra/config/setup_kitp_greenland.toml
@@ -1,22 +1,33 @@
ismip7_to_pism = {"acabf" = "climatic_mass_balance", "runoff" = "water_input_rate", "tas" = "ice_surface_temp", "TF" = "theta_ocean", "geothermal_heat_flux1" = "bheatflx"}
-gcms = ["CESM1-WACCM-SC"]
-version = "v1"
+version = "v4"
bucket = "pism-cloud-data"
-prefix = "kitp_greenland_input"
+prefix = "kitp/input"
+baseline = "hirham5"
[domain]
-x_bounds = [-653000.0, 879700.0]
-y_bounds = [-3384350.0, -632750.0]
+crs = "EPSG:3413"
+x_bounds = [-750650.0, 977350.0]
+y_bounds = [-3412550.0, -604550.0]
resolution = "900m"
[forcing]
bucket = "pism-cloud-data"
-prefix = "kitp_forcing"
-present_day_forcings = ["pdSST-pdSIC", "pdSST-pdSICSIT", "pa-pdSIC-ext"]
-future_forcings = ["futSST-pdSIC", "pdSST-futArcSIC", "pa-futArcSIC-ext"]
+prefix = "kitp/kitp_forcing_v4"
-[pathway]
+[gcms]
-baseline = {"start_year" = 1990, "end_year" = 2019}
+'AWI-CM-1-1-MR' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CESM1-WACCM-SC' = [["pdSST-futArcSICSIT", "pdSST-pdSICSIT"], ["pa-futArcSIC-ext", "pa-pdSIC-ext"], ["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CESM2' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CNRM-CM6-1' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'CanESM5' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'HadGEM3-GC31-MM' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+'IPSL-CM6A-LR' = [["futSST-pdSIC", "pdSST-pdSIC"], ["pdSST-futArcSIC", "pdSST-pdSIC"], ["futSST-futArcSIC-SUM", "pdSST-pdSIC"]]
+
+
+[climatology]
+
+hirham5 = {"start_year" = 1990, "end_year" = 2019}
+carra2 = {"year" = [1986, 1987, 1988, 1991, 1992, 1993, 1996, 1997, 1998, 2001, 2002, 2003, 2006, 2007, 2008, 2011, 2012, 2013, 2016, 2017, 2018, 2021, 2022, 2023]}
diff --git a/pism_terra/config/setup_rgi.toml b/pism_terra/config/setup_rgi.toml
new file mode 100644
index 0000000..3724a02
--- /dev/null
+++ b/pism_terra/config/setup_rgi.toml
@@ -0,0 +1,21 @@
+[regions]
+
+1 = {name = "alaska"}
+2 = {name = "western_canada_usa"}
+3 = {name = "arctic_canada_north"}
+4 = {name = "arctic_canada_south"}
+5 = {name = "greenland_periphery"}
+6 = {name = "iceland"}
+7 = {name = "svalbard_jan_mayen"}
+8 = {name = "scandinavia"}
+9 = {name = "russian_arctic"}
+10 = {name = "north_asia"}
+11 = {name = "central_europe"}
+12 = {name = "caucasus_middle_east"}
+13 = {name = "central_asia"}
+14 = {name = "south_asia_west"}
+15 = {name = "south_asia_east"}
+16 = {name = "low_latitudes"}
+17 = {name = "southern_andes"}
+18 = {name = "new_zealand"}
+19 = {name = "subantarctic_antarctic_islands"}
diff --git a/pism_terra/config/setup_s4f.toml b/pism_terra/config/setup_s4f.toml
new file mode 100644
index 0000000..3ebde69
--- /dev/null
+++ b/pism_terra/config/setup_s4f.toml
@@ -0,0 +1,5 @@
+[regions]
+
+1 = {name = "alaska", crs = "EPSG:5936"}
+3 = {name = "arctic_canada_north", crs = "EPSG:3413"}
+7 = {name = "svalbard_jan_mayen", crs = "EPSG:3413"}
diff --git a/pism_terra/config/test_bm_greenland.toml b/pism_terra/config/test_bm_greenland.toml
deleted file mode 100644
index 3486030..0000000
--- a/pism_terra/config/test_bm_greenland.toml
+++ /dev/null
@@ -1,18 +0,0 @@
-ismip7_to_pism = {"acabf" = "climatic_mass_balance", "runoff" = "water_input_rate", "tas" = "ice_surface_temp", "TF" = "theta_ocean", "geothermal_heat_flux1" = "bheatflx"}
-gcms = ["CESM2-WACCM"]
-
-[domain]
-
-x_bounds = [-653000.0, 879700.0]
-y_bounds = [-3384350.0, -632750.0]
-resolution = "2400m"
-
-[pathway]
-
-historical = {"start_year" = 1980, "end_year" = 1981, version = 1}
-ssp585 = {"start_year" = 2015, "end_year" = 2016, version = 1 }
-
-[forcing]
-
-climate = {fields = ["acabf", "runoff", "tas"], short_hand = "SDBN1"}
-ocean = {fields = ["TF"], short_hand = "none"}
diff --git a/pism_terra/config/test_bm_greenland_historical.toml b/pism_terra/config/test_bm_greenland_historical.toml
deleted file mode 100644
index 3be69af..0000000
--- a/pism_terra/config/test_bm_greenland_historical.toml
+++ /dev/null
@@ -1,203 +0,0 @@
-[run]
-
-executable = "pism"
-mpi = "mpirun -np {{ ntasks }}"
-ntasks = 24
-writer = ""
-
-[job]
-
-queue = "t2small"
-walltime = "48:00:00"
-nodes = 1
-
-[run_info]
-
-'run_info.institution' = "University of Alaska Fairbanks"
-'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
-
-[campaign]
-
-bucket = "pism-cloud-data"
-prefix = "bedmachine_greenland_input_testing"
-pathway = "historical"
-version = "1"
-start_year = "1980"
-end_year = "1981"
-gcm = "CESM2-WACCM"
-boot_file = "boot_g2400m_GreenlandObsISMIP7-v1.3.nc"
-grid_file = "ismip7_greenland_grid.nc"
-regrid_file = "g2400m_id_BAYES-MEDIAN_1980-1-1_1984-12-31.nc"
-retreat_file = "pism_g2400m_frontretreat_calfin_1972_2019_ME.nc"
-heatflux_file = "heatflux_g2400m_GreenlandObsISMIP7-v1.3.nc"
-
-[time]
-
-'time.start' = "1980-01-01"
-'time.end' = "1981-01-01"
-'time.calendar' = "standard"
-'time.reference_date' = "1978-01-01"
-
-[time_stepping]
-
-'time_stepping.skip.enabled' = "yes"
-'time_stepping.skip.max' = 100
-
-['geometry']
-
-'geometry.front_retreat.use_cfl' = 'yes'
-'geometry.part_grid.enabled' = 'yes'
-'geometry.remove_icebergs' = 'yes'
-'geometry.front_retreat.prescribed.file' = "none"
-
-['input']
-
-'input.bootstrap' = "yes"
-'input.forcing.time_extrapolation' = "yes"
-'input.forcing.buffer_size' = 37
-'input.regrid.file' = "none"
-'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume"
-
-[reporting]
-
-'output.format' = "netcdf4_serial"
-'output.compression_level' = 2
-'output.size' = "medium"
-'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
-'output.scalar.file' = "none"
-'output.scalar.times' = 'daily'
-'output.spatial.file' = "none"
-'output.spatial.times' = "monthly"
-'output.spatial.vars' = "bmelt,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,tauc,tillwat"
-'output.checkpoint.interval' = 24
-'output.extra.stop_missing' = 'no'
-'output.ISMIP6' = "no"
-
-[grid]
-
-resolution = "2400m"
-'grid.Lbz' = 2000
-'grid.Lz' = 4000
-'grid.Mbz' = 51
-'grid.Mz' = 101
-'grid.registration' = "center"
-'grid.allow_extrapolation' = "yes"
-
-[energy]
-
-model = "enthalpy"
-
-[energy.options.enthalpy]
-
-'energy.model' = "enthalpy"
-'energy.bedrock_thermal.file' = "none"
-'stress_balance.blatter.flow_law' = "gpbld"
-'stress_balance.sia.flow_law' = "gpbld"
-'stress_balance.ssa.flow_law' = "gpbld"
-
-[stress_balance]
-
-model = 'blatter'
-
-[stress_balance.options.blatter]
-
-'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.max_diffusivity' = 1000000.0
-'time_stepping.adaptive_ratio' = 100
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
-
-[stress_balance.options.hybrid]
-
-'stress_balance.model' = "ssa+sia"
-'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.max_diffusivity' = 100000.0
-
-[iceflow]
-
-'basal_resistance.pseudo_plastic.enabled' = 'yes'
-'basal_resistance.pseudo_plastic.q' = 0.7508221
-'basal_yield_stress.model' = 'mohr_coulomb'
-'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
-'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
-'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
-'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
-'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
-'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
-'stress_balance.blatter.enhancement_factor' = 2.608046
-'stress_balance.sia.enhancement_factor' = 2.608046
-'stress_balance.ssa.Glen_exponent' = 3.309718
-
-[surface]
-
-model = "given"
-
-[surface.options.given]
-
-'surface.models' = "given"
-'surface.given.file' = "none"
-
-[calving]
-
-'calving.methods' = 'vonmises_calving'
-'calving.vonmises_calving.sigma_max' = 7.5e5
-
-[atmosphere]
-
-model = "given"
-
-[atmosphere.options.given]
-
-'atmosphere.models' = "given"
-'atmosphere.given.file' = "none"
-
-[ocean]
-
-'ocean.models' = "th"
-'ocean.th.file' = "none"
-'ocean.th.clip_salinity' = "False"
-
-[hydrology]
-
-model = "routing"
-
-[hydrology.options.null]
-
-'hydrology.model' = "null"
-'hydrology.null_diffuse_till_water' = "yes"
-
-[hydrology.options.routing]
-
-'hydrology.model' = "routing"
-'hydrology.routing.include_floating_ice' = "yes"
-'hydrology.surface_input.file' = "none"
-
-
-[frontal_melt]
-
-model = "routing"
-
-[frontal_melt.options.off]
-
-[frontal_melt.options.routing]
-
-'frontal_melt.routing.file' = "none"
-'frontal_melt.routing.parameter_a' = 3e-4
-'frontal_melt.routing.parameter_b' = 0.15
-'frontal_melt.routing.power_alpha' = 0.39
-'frontal_melt.routing.power_beta' = 1.18
diff --git a/pism_terra/config/test_ismip7_greenland.toml b/pism_terra/config/test_ismip7_greenland.toml
deleted file mode 100644
index 65b401d..0000000
--- a/pism_terra/config/test_ismip7_greenland.toml
+++ /dev/null
@@ -1,20 +0,0 @@
-ismip7_to_pism = {"acabf" = "climatic_mass_balance", "runoff" = "water_input_rate", "tas" = "ice_surface_temp", "TF" = "theta_ocean", "geothermal_heat_flux1" = "bheatflx"}
-gcms = ["CESM2-WACCM"]
-bucket = "pism-cloud-data"
-prefix = "ismip7_greenland_input_testing"
-
-[domain]
-
-x_bounds = [-720500.0, 960500.0]
-y_bounds = [-3450500.0, -569500.0]
-resolution = "1000m"
-
-[pathway]
-
-historical = {"start_year" = 1980, "end_year" = 1981, version = 1}
-ssp585 = {"start_year" = 2015, "end_year" = 2016, version = 1 }
-
-[forcing]
-
-climate = {fields = ["acabf", "runoff", "tas"], short_hand = "SDBN1"}
-ocean = {fields = ["TF"], short_hand = "none"}
diff --git a/pism_terra/config/test_ismip7_greenland_historical.toml b/pism_terra/config/test_ismip7_greenland_historical.toml
deleted file mode 100644
index 0831734..0000000
--- a/pism_terra/config/test_ismip7_greenland_historical.toml
+++ /dev/null
@@ -1,201 +0,0 @@
-[run]
-
-executable = "pism"
-mpi = "mpirun -np {{ ntasks }}"
-ntasks = 8
-writer = ""
-
-[job]
-
-queue = "t2small"
-walltime = "48:00:00"
-nodes = 1
-
-[run_info]
-
-'run_info.institution' = "University of Alaska Fairbanks"
-'run_info.title' = "PISM-TERRA ISMIP7 Greenland Historical Run"
-
-[campaign]
-
-bucket = "pism-cloud-data"
-prefix = "ismip7_greenland_input_testing"
-pathway = "historical"
-version = "1"
-start_year = "1980"
-end_year = "1981"
-gcm = "CESM2-WACCM"
-boot_file = "boot_g1000m_GreenlandObsISMIP7-v1.3.nc"
-grid_file = "ismip7_greenland_grid.nc"
-regrid_file = "g1200m_id_BAYES-MEDIAN_1980-1-1_1984-12-31.nc"
-retreat_file = "pism_g1000m_frontretreat_calfin_1972_2019_ME.nc"
-heatflux_file = "heatflux_GreenlandObsISMIP7-v1.3.nc"
-
-[time]
-
-'time.start' = "1980-01-01"
-'time.end' = "1980-01-20"
-'time.calendar' = "standard"
-'time.reference_date' = "1978-01-01"
-
-[time_stepping]
-
-'time_stepping.adaptive_ratio' = 250
-'time_stepping.skip.enabled' = "yes"
-'time_stepping.skip.max' = 100
-
-['geometry']
-
-'geometry.front_retreat.use_cfl' = 'yes'
-'geometry.part_grid.enabled' = 'yes'
-'geometry.remove_icebergs' = 'yes'
-'geometry.front_retreat.prescribed.file' = "none"
-
-['input']
-
-'input.bootstrap' = "yes"
-'input.forcing.time_extrapolation' = "yes"
-'input.forcing.buffer_size' = 37
-'input.regrid.file' = "none"
-'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume"
-
-[reporting]
-
-'output.format' = "netcdf4_serial"
-'output.compression_level' = 2
-'output.size' = "medium"
-'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
-'output.extra.file' = "none"
-'output.extra.times' = "daily"
-'output.extra.vars' = "bmelt,effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,tauc,tillwat,diffusivity"
-'output.timeseries.filename' = "none"
-'output.timeseries.times' = "daily"
-'output.checkpoint.interval' = 24
-'output.extra.stop_missing' = 'no'
-'output.ISMIP6' = "no"
-
-[grid]
-
-resolution = "2000m"
-'grid.Lbz' = 2000
-'grid.Lz' = 4000
-'grid.Mbz' = 51
-'grid.Mz' = 101
-'grid.registration' = "center"
-'grid.allow_extrapolation' = "yes"
-
-[energy]
-
-model = "enthalpy"
-
-[energy.options.enthalpy]
-
-'energy.model' = "enthalpy"
-'energy.bedrock_thermal.file' = "none"
-'stress_balance.blatter.flow_law' = "gpbld"
-'stress_balance.sia.flow_law' = "gpbld"
-'stress_balance.ssa.flow_law' = "gpbld"
-
-[stress_balance]
-
-model = 'hybrid'
-
-[stress_balance.options.blatter]
-
-'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.max_diffusivity' = 1000000.0
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
-
-[stress_balance.options.hybrid]
-
-'stress_balance.model' = "ssa+sia"
-'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.max_diffusivity' = 100000.0
-
-[iceflow]
-
-'basal_resistance.pseudo_plastic.enabled' = 'yes'
-'basal_resistance.pseudo_plastic.q' = 0.7508221
-'basal_yield_stress.model' = 'mohr_coulomb'
-'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
-'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
-'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
-'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
-'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
-'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
-'stress_balance.blatter.enhancement_factor' = 2.608046
-'stress_balance.sia.enhancement_factor' = 2.608046
-'stress_balance.ssa.Glen_exponent' = 3.309718
-
-[surface]
-
-model = "given"
-
-[surface.options.given]
-
-'surface.models' = "given"
-'surface.given.file' = "none"
-
-[calving]
-
-'calving.methods' = 'vonmises_calving'
-'calving.vonmises_calving.sigma_max' = 7.5e5
-
-[atmosphere]
-
-model = "given"
-
-[atmosphere.options.given]
-
-'atmosphere.models' = "given"
-'atmosphere.given.file' = "none"
-
-[ocean]
-
-'ocean.models' = "th"
-'ocean.th.file' = "none"
-'ocean.th.clip_salinity' = "False"
-
-[hydrology]
-
-model = "routing"
-
-[hydrology.options.null]
-
-'hydrology.model' = "null"
-'hydrology.null_diffuse_till_water' = "yes"
-
-[hydrology.options.routing]
-
-'hydrology.model' = "routing"
-'hydrology.routing.include_floating_ice' = "yes"
-'hydrology.surface_input.file' = "none"
-
-
-[frontal_melt]
-
-model = "routing"
-
-[frontal_melt.options.routing]
-
-'frontal_melt.routing.file' = "none"
-'frontal_melt.routing.parameter_a' = 3e-4
-'frontal_melt.routing.parameter_b' = 0.15
-'frontal_melt.routing.power_alpha' = 0.39
-'frontal_melt.routing.power_beta' = 1.18
diff --git a/pism_terra/config/test_kitp_greenland_long.toml b/pism_terra/config/test_kitp_greenland_long.toml
deleted file mode 100644
index 7eca457..0000000
--- a/pism_terra/config/test_kitp_greenland_long.toml
+++ /dev/null
@@ -1,197 +0,0 @@
-[run]
-
-executable = "pism"
-mpi = "mpirun -np {{ ntasks }}"
-ntasks = 8
-writer = ""
-
-[job]
-
-queue = "t2small"
-walltime = "4:00:00"
-nodes = 1
-
-[run_info]
-
-'run_info.institution' = "University of Alaska Fairbanks"
-'run_info.title' = "PISM-TERRA KITP Greenland Sea Ice Run"
-
-[campaign]
-
-bucket = "pism-cloud-data"
-prefix = "kitp/input_testing"
-version = "v1"
-gcms = "CESM1-WACCM-SC"
-present_day_forcings = ["pdSST-pdSIC", "pdSST-pdSICSIT", "pa-pdSIC-ext"]
-future_forcings = ["futSST-pdSIC", "pdSST-futArcSIC", "pa-futArcSIC-ext"]
-boot_file = "boot_g2400m_GreenlandObsISMIP7-v1.3.nc"
-grid_file = "ismip7_greenland_grid.nc"
-regrid_file = "g2400m_id_BAYES-MEDIAN_2007-01-01_2010-01-01.nc"
-heatflux_file = "heatflux_g2400m_GreenlandObsISMIP7-v1.3.nc"
-outline_file = "Greenland_Basins_PS_v1.4.2_w_shelves.gpkg"
-
-[time]
-
-'time.start' = "0001-01-01"
-'time.end' = "0300-01-01"
-'time.calendar' = "365_day"
-'time.reference_date' = "0001-01-01"
-
-[time_stepping]
-
-'time_stepping.adaptive_ratio' = 0.12
-'time_stepping.skip.enabled' = "yes"
-'time_stepping.skip.max' = 100
-
-['geometry']
-
-'geometry.front_retreat.use_cfl' = 'yes'
-'geometry.part_grid.enabled' = 'yes'
-'geometry.remove_icebergs' = 'yes'
-
-['input']
-
-'input.bootstrap' = "yes"
-'input.forcing.time_extrapolation' = "yes"
-'input.forcing.buffer_size' = 37
-'input.regrid.file' = "none"
-'input.regrid.vars' = "litho_temp,enthalpy,age,tillwat,bmelt,ice_area_specific_volume"
-
-[reporting]
-
-'output.format' = "netcdf4_serial"
-'output.compression_level' = 2
-'output.size' = "medium"
-'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf"
-'output.spatial.file' = "none"
-'output.spatial.times' = "yearly"
-'output.spatial.vars' = "bmelt,effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,mask,usurf,tauc,tillwat,diffusivity,ice_mass,mass_fluxes"
-'output.scalar.file' = "none"
-'output.scalar.times' = "monthly"
-'output.checkpoint.interval' = 24
-'output.extra.stop_missing' = 'no'
-'output.ISMIP6' = "no"
-'output.experiment_id_max_length' = 100
-
-[grid]
-
-'resolution' = "2400m"
-'grid.Lbz' = 2000
-'grid.Lz' = 4000
-'grid.Mbz' = 51
-'grid.Mz' = 101
-'grid.registration' = "center"
-'grid.allow_extrapolation' = "yes"
-
-[energy]
-
-model = "enthalpy"
-
-[energy.options.enthalpy]
-
-'energy.model' = "enthalpy"
-'energy.bedrock_thermal.file' = "none"
-'stress_balance.blatter.flow_law' = "gpbld"
-'stress_balance.sia.flow_law' = "gpbld"
-'stress_balance.ssa.flow_law' = "gpbld"
-
-[stress_balance]
-
-model = 'hybrid'
-
-[stress_balance.options.blatter]
-
-'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.max_diffusivity' = 1000000.0
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
-
-[stress_balance.options.hybrid]
-
-'stress_balance.model' = "ssa+sia"
-'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.max_diffusivity' = 100000.0
-
-[iceflow]
-
-'basal_resistance.pseudo_plastic.enabled' = 'yes'
-'basal_resistance.pseudo_plastic.q' = 0.7508221
-'basal_yield_stress.model' = 'mohr_coulomb'
-'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.01845403
-'basal_yield_stress.mohr_coulomb.topg_to_phi.enabled' = "yes"
-'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_max' = 42.79528
-'basal_yield_stress.mohr_coulomb.topg_to_phi.phi_min' = 7.193718
-'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_max' = 243.8239
-'basal_yield_stress.mohr_coulomb.topg_to_phi.topg_min' = -369.6359
-'stress_balance.blatter.enhancement_factor' = 2.608046
-'stress_balance.sia.enhancement_factor' = 2.608046
-'stress_balance.ssa.Glen_exponent' = 3.309718
-
-[surface]
-
-model = "pdd"
-
-[surface.options.pdd]
-
-'surface.models' = "pdd"
-
-[calving]
-
-'calving.methods' = 'vonmises_calving'
-'calving.vonmises_calving.sigma_max' = 7.5e5
-
-[atmosphere]
-
-model = "given"
-
-[atmosphere.options.given]
-
-'atmosphere.models' = "given"
-'atmosphere.given.file' = "none"
-
-[ocean]
-
-[hydrology]
-
-model = "null"
-
-[hydrology.options.null]
-
-'hydrology.model' = "null"
-'hydrology.null_diffuse_till_water' = "yes"
-
-[hydrology.options.routing]
-
-'hydrology.model' = "routing"
-'hydrology.routing.include_floating_ice' = "yes"
-'hydrology.surface_input.file' = "none"
-
-
-[frontal_melt]
-
-model = "off"
-
-[frontal_melt.options.off]
-
-[frontal_melt.options.routing]
-
-'frontal_melt.routing.file' = "none"
-'frontal_melt.routing.parameter_a' = 3e-4
-'frontal_melt.routing.parameter_b' = 0.15
-'frontal_melt.routing.power_alpha' = 0.39
-'frontal_melt.routing.power_beta' = 1.18
diff --git a/pism_terra/config/test_rgi.toml b/pism_terra/config/test_rgi.toml
deleted file mode 100644
index 647108e..0000000
--- a/pism_terra/config/test_rgi.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-[regions]
-
-1 = "alaska"
diff --git a/pism_terra/config/test_setup_kitp_greenland.toml b/pism_terra/config/test_setup_kitp_greenland.toml
deleted file mode 100644
index ee6554c..0000000
--- a/pism_terra/config/test_setup_kitp_greenland.toml
+++ /dev/null
@@ -1,22 +0,0 @@
-ismip7_to_pism = {"acabf" = "climatic_mass_balance", "runoff" = "water_input_rate", "tas" = "ice_surface_temp", "TF" = "theta_ocean", "geothermal_heat_flux1" = "bheatflx"}
-gcms = ["CESM1-WACCM-SC"]
-version = "v1"
-bucket = "pism-cloud-data"
-prefix = "kitp_greenland_input_testing"
-
-[domain]
-
-x_bounds = [-653000.0, 879700.0]
-y_bounds = [-3384350.0, -632750.0]
-resolution = "2400m"
-
-[forcing]
-
-bucket = "pism-cloud-data"
-prefix = "kitp_forcing"
-present_day_forcings = ["pdSST-pdSIC", "pdSST-pdSICSIT", "pa-pdSIC-ext"]
-future_forcings = ["futSST-pdSIC", "pdSST-futArcSIC", "pa-futArcSIC-ext"]
-
-[pathway]
-
-baseline = {"start_year" = 1990, "end_year" = 2019}
diff --git a/pism_terra/config/test_wrangell_local.toml b/pism_terra/config/test_wrangell_local.toml
deleted file mode 100644
index 8c96583..0000000
--- a/pism_terra/config/test_wrangell_local.toml
+++ /dev/null
@@ -1,190 +0,0 @@
-[run]
-
-executable = "pism"
-mpi = ""
-ntasks = 8
-
-[job]
-
-queue = "t2small"
-walltime = "72:00:00"
-nodes = 1
-
-[run_info]
-
-'run_info.institution' = "University of Idaho"
-'run_info.title' = "PISM-TERRA Wrangell Mountains"
-
-[campaign]
-
-bucket = "pism-cloud-data"
-dem = "glo_30"
-climate = "snap"
-ice_thickness = "maffezzoli"
-name = "Equilibrium"
-prefix = "glacier"
-rgi_file = "rgi_c.gpkg"
-velocity = "its_live"
-
-[time]
-
-'time.start' = "01-01-01"
-'time.end' = "02-01-01"
-'time.calendar' = "365_day"
-'time.reference_date' = "01-01-01"
-
-[time_stepping]
-
-'time_stepping.adaptive_ratio' = 250
-'time_stepping.skip.enabled' = "yes"
-'time_stepping.skip.max' = 100
-
-['geometry']
-
-'geometry.front_retreat.use_cfl' = 'yes'
-'geometry.part_grid.enabled' = 'yes'
-'geometry.remove_icebergs' = 'yes'
-
-['input']
-
-'input.bootstrap' = "yes"
-'input.forcing.time_extrapolation' = "yes"
-'input.forcing.buffer_size' = 13
-
-[reporting]
-
-'output.format' = "netcdf4_parallel"
-'output.compression_level'= 2
-'output.size' = "medium"
-'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf,bmelt"
-'output.scalar.times' = "daily"
-'output.spatial.file' = "none"
-'output.spatial.times' = "monthly"
-'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,ice_mass,mass_fluxes,mask,usurf"
-'output.checkpoint.interval' = 24
-'output.spatial.stop_missing' = 'no'
-
-[grid]
-
-resolution = "1000m"
-grid.Lbz = 0
-grid.Lz = 2000
-grid.Mbz = 1
-grid.Mz = 201
-grid.registration = "center"
-
-[energy]
-
-model = "enthalpy"
-
-[energy.options.none]
-
-'energy.model' = "none"
-'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
-'stress_balance.blatter.flow_law' = "isothermal_glen"
-'stress_balance.sia.flow_law' = "isothermal_glen"
-'stress_balance.ssa.flow_law' = "isothermal_glen"
-
-[energy.options.enthalpy]
-
-'energy.model' = "enthalpy"
-'stress_balance.blatter.flow_law' = "gpbld"
-'stress_balance.sia.flow_law' = "gpbld"
-'stress_balance.ssa.flow_law' = "gpbld"
-
-[stress_balance]
-
-model = 'blatter'
-
-[stress_balance.options.blatter]
-
-'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
-'stress_balance.blatter.use_eta_transform' = 'yes'
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.blatter.relative_convergence' = 1.0e-3
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
-
-[stress_balance.options.hybrid]
-
-'stress_balance.model' = "ssa+sia"
-'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.surface_gradient_method' = 'eta'
-'stress_balance.sia.max_diffusivity' = 100000.0
-'stress_balance.ssa.fd.relative_convergence' = 1.0e-3
-
-[stress_balance.options.sia]
-
-'stress_balance.model' = "sia"
-'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.surface_gradient_method' = 'eta'
-'stress_balance.sia.max_diffusivity' = 100000.0
-
-[iceflow]
-
-'basal_resistance.pseudo_plastic.q' = 0.75
-'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
-'basal_resistance.pseudo_plastic.enabled' = 'yes'
-'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
-'basal_yield_stress.model' = 'mohr_coulomb'
-'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
-'stress_balance.blatter.enhancement_factor' = 2.0
-'stress_balance.sia.enhancement_factor' = 2.0
-
-[surface]
-
-model = "pdd"
-
-[surface.options.pdd]
-
-'surface.models' = 'pdd'
-
-[calving]
-
-'calving.methods' = 'float_kill'
-
-[atmosphere]
-
-model = "given"
-
-[atmosphere.options.given]
-
-'atmosphere.models' = "given,delta_T,precip_scaling"
-'atmosphere.given.file' = "none"
-'atmosphere.given.periodic' = ""
-'atmosphere.elevation_change.temperature_lapse_rate' = 6
-'atmosphere.delta_T.file' = "none"
-'atmosphere.elevation_change.file' = "none"
-'atmosphere.frac_P.file' = "none"
-
-[ocean]
-
-'ocean.constant.melt_rate' = 0.0
-'ocean.models' = 'constant'
-
-[hydrology]
-
-model = "null"
-
-[hydrology.options]
-
-[frontal_melt]
-
-model = "off"
-
-[frontal_melt.options.off]
diff --git a/pism_terra/config/wrangell_chinook.toml b/pism_terra/config/wrangell_chinook.toml
deleted file mode 100644
index d0ea78d..0000000
--- a/pism_terra/config/wrangell_chinook.toml
+++ /dev/null
@@ -1,182 +0,0 @@
-[run]
-
-executable = "pism"
-mpi = ""
-ntasks = 8
-
-[job]
-
-queue = "t2small"
-walltime = "72:00:00"
-nodes = 1
-
-[run_info]
-
-'run_info.institution' = "University of Idaho"
-'run_info.title' = "PISM-TERRA Wrangell Mountains"
-
-[campaign]
-
-name = "Equilibrium"
-bucket = "pism-cloud-data"
-climate = "snap"
-dem = "glo_30"
-ice_thickness = "maffezzoli"
-velocity = "its_live"
-
-[time]
-
-'time.start' = "01-01-01"
-'time.end' = "05-01-01"
-'time.calendar' = "365_day"
-'time.reference_date' = "01-01-01"
-
-[time_stepping]
-
-'time_stepping.adaptive_ratio' = 250
-'time_stepping.skip.enabled' = "yes"
-'time_stepping.skip.max' = 100
-
-['geometry']
-
-'geometry.front_retreat.use_cfl' = 'yes'
-'geometry.part_grid.enabled' = 'yes'
-'geometry.remove_icebergs' = 'yes'
-
-['input']
-
-'input.bootstrap' = "yes"
-'input.forcing.time_extrapolation' = "yes"
-'input.forcing.buffer_size' = 13
-
-[reporting]
-
-'output.format' = "netcdf4_parallel"
-'output.compression_level'= 2
-'output.size' = "medium"
-'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf,bmelt"
-'output.scalar.times' = "daily"
-'output.spatial.file' = "none"
-'output.spatial.times' = 1
-'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,ice_mass,mass_fluxes,mask,usurf"
-'output.checkpoint.interval' = 24
-'output.spatial.stop_missing' = 'no'
-
-[grid]
-
-resolution = "500m"
-grid.Lbz = 0
-grid.Lz = 2000
-grid.Mbz = 1
-grid.Mz = 201
-grid.registration = "center"
-
-[energy]
-
-model = "enthalpy"
-
-[energy.options.none]
-
-'energy.model' = "none"
-'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
-'stress_balance.blatter.flow_law' = "isothermal_glen"
-'stress_balance.sia.flow_law' = "isothermal_glen"
-'stress_balance.ssa.flow_law' = "isothermal_glen"
-
-[energy.options.enthalpy]
-
-'energy.model' = "enthalpy"
-'stress_balance.blatter.flow_law' = "gpbld"
-'stress_balance.sia.flow_law' = "gpbld"
-'stress_balance.ssa.flow_law' = "gpbld"
-
-[stress_balance]
-
-model = 'blatter'
-
-[stress_balance.options.blatter]
-
-'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
-'stress_balance.blatter.use_eta_transform' = 'yes'
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.blatter.relative_convergence' = 1.0e-3
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
-
-[stress_balance.options.hybrid]
-
-'stress_balance.model' = "ssa+sia"
-'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.surface_gradient_method' = 'eta'
-'stress_balance.sia.max_diffusivity' = 100000.0
-'stress_balance.ssa.fd.relative_convergence' = 1.0e-3
-
-[stress_balance.options.sia]
-
-'stress_balance.model' = "sia"
-'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.surface_gradient_method' = 'eta'
-'stress_balance.sia.max_diffusivity' = 100000.0
-
-[iceflow]
-
-'basal_resistance.pseudo_plastic.q' = 0.75
-'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
-'basal_resistance.pseudo_plastic.enabled' = 'yes'
-'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
-'basal_yield_stress.model' = 'mohr_coulomb'
-'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
-'stress_balance.blatter.enhancement_factor' = 2.0
-'stress_balance.sia.enhancement_factor' = 2.0
-
-[surface]
-
-model = "pdd"
-
-[surface.options.pdd]
-
-'surface.models' = 'pdd'
-
-[calving]
-
-'calving.methods' = 'float_kill'
-
-[atmosphere]
-
-model = "given"
-
-[atmosphere.options.given]
-
-'atmosphere.models' = "given,delta_T,precip_scaling"
-'atmosphere.given.file' = "none"
-'atmosphere.given.periodic' = ""
-'atmosphere.elevation_change.temperature_lapse_rate' = 6
-'atmosphere.delta_T.file' = "none"
-'atmosphere.elevation_change.file' = "none"
-'atmosphere.frac_P.file' = "none"
-
-[ocean]
-
-'ocean.constant.melt_rate' = 0.0
-'ocean.models' = 'constant'
-
-[hydrology]
-
-model = "null"
-
-[hydrology.options]
diff --git a/pism_terra/config/wrangell_chinook_dbg.toml b/pism_terra/config/wrangell_chinook_dbg.toml
deleted file mode 100644
index d9fcad4..0000000
--- a/pism_terra/config/wrangell_chinook_dbg.toml
+++ /dev/null
@@ -1,175 +0,0 @@
-[run]
-
-executable = "pism"
-mpi = "mpirun -np {{ ntasks }}"
-ntasks = 8
-
-[job]
-
-queue = "t2small"
-walltime = "7:00:00"
-nodes = 1
-
-[run_info]
-
-'run_info.institution' = "University of Alaska Fairbanks"
-'run_info.title' = "PISM-TERRA Atna"
-
-[campaign]
-
-name = "LGM"
-climate = "snap"
-dem = "glo_30"
-ice_thickness = "millan"
-velocity = "none"
-
-[time]
-
-'time.start' = "01-01-01"
-'time.end' = "01-04-01"
-'time.calendar' = "365_day"
-'time.reference_date' = "01-01-01"
-
-[time_stepping]
-
-'time_stepping.adaptive_ratio' = 250
-'time_stepping.skip.enabled' = "yes"
-'time_stepping.skip.max' = 100
-
-['geometry']
-
-'geometry.front_retreat.use_cfl' = 'yes'
-'geometry.part_grid.enabled' = 'yes'
-'geometry.remove_icebergs' = 'yes'
-
-['input']
-
-'input.bootstrap' = "yes"
-'input.forcing.time_extrapolation' = "yes"
-'input.forcing.buffer_size' = 13
-
-[reporting]
-
-'output.format' = "netcdf4_parallel"
-'output.compression_level'= 2
-'output.size' = "medium"
-'output.sizes.medium' = "sftgif,velsurf_mag,mask,usurf,bmelt"
-'output.scalar.times' = "daily"
-'output.spatial.file' = "none"
-'output.spatial.times' = "monthly"
-'output.spatial.vars' = "effective_air_temp,effective_precipitation,ice_surface_temp,climatic_mass_balance,thk,velsurf_mag,velbase_mag,ice_mass,mass_fluxes,mask,usurf"
-'output.checkpoint.interval' = 24
-'output.spatial.stop_missing' = 'no'
-
-[grid]
-
-resolution = "500m"
-grid.Lbz = 0
-grid.Lz = 5000
-grid.Mbz = 1
-grid.Mz = 251
-grid.registration = "center"
-
-[energy]
-
-model = "enthalpy"
-
-[energy.options.none]
-
-'energy.model' = "none"
-'flow_law.isothermal_Glen.ice_softness' = 1.0e-24
-'stress_balance.blatter.flow_law' = "isothermal_glen"
-'stress_balance.sia.flow_law' = "isothermal_glen"
-'stress_balance.ssa.flow_law' = "isothermal_glen"
-
-[energy.options.enthalpy]
-
-'energy.model' = "enthalpy"
-'stress_balance.blatter.flow_law' = "gpbld"
-'stress_balance.sia.flow_law' = "gpbld"
-'stress_balance.ssa.flow_law' = "gpbld"
-
-[stress_balance]
-
-model = 'blatter'
-
-[stress_balance.options.blatter]
-
-'stress_balance.model' = "blatter"
-'stress_balance.blatter.Mz' = 17
-'stress_balance.blatter.coarsening_factor' = 4
-'stress_balance.blatter.use_eta_transform' = 'yes'
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.blatter.relative_convergence' = 1.0e-3
-bp_ksp_monitor = ""
-bp_ksp_view_singularvalues = ""
-bp_snes_monitor_ratio = ""
-bp_pc_type = "mg"
-bp_mg_levels_ksp_type = "richardson"
-bp_mg_levels_pc_type ="sor"
-bp_mg_coarse_ksp_type = "preonly"
-bp_mg_coarse_pc_type = "lu"
-bp_pc_mg_levels = 3
-bp_snes_ksp_ew = 1
-bp_snes_ksp_ew_version = 3
-bp_snes_rtol = 1e-3
-bp_ksp_rtol = 1e-3
-
-[stress_balance.options.hybrid]
-
-'stress_balance.model' = "ssa+sia"
-'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.surface_gradient_method' = 'eta'
-'stress_balance.sia.max_diffusivity' = 100000.0
-'stress_balance.ssa.fd.relative_convergence' = 1.0e-3
-
-[stress_balance.options.sia]
-
-'stress_balance.model' = "sia"
-'stress_balance.ssa.method' = "fd"
-'stress_balance.calving_front_stress_bc' = 'yes'
-'stress_balance.sia.surface_gradient_method' = 'eta'
-'stress_balance.sia.max_diffusivity' = 100000.0
-
-[iceflow]
-
-'basal_resistance.pseudo_plastic.q' = 0.75
-'basal_resistance.pseudo_plastic.u_threshold' = '100m/yr'
-'basal_resistance.pseudo_plastic.enabled' = 'yes'
-'basal_yield_stress.mohr_coulomb.till_phi_default' = 30
-'basal_yield_stress.model' = 'mohr_coulomb'
-'basal_yield_stress.mohr_coulomb.till_effective_fraction_overburden' = 0.025
-'stress_balance.blatter.enhancement_factor' = 2.0
-'stress_balance.sia.enhancement_factor' = 2.0
-
-[surface]
-
-model = "pdd"
-
-[surface.options.pdd]
-
-'surface.models' = 'pdd'
-
-[calving]
-
-'calving.methods' = 'float_kill'
-
-[atmosphere]
-
-model = "given"
-
-[atmosphere.options.given]
-
-'atmosphere.models' = "given,delta_T,precip_scaling"
-'atmosphere.given.file' = "none"
-'atmosphere.given.periodic' = ""
-'atmosphere.elevation_change.temperature_lapse_rate' = 6
-'atmosphere.delta_T.file' = "none"
-'atmosphere.elevation_change.file' = "none"
-'atmosphere.frac_P.file' = "none"
-
-[ocean]
-
-'ocean.constant.melt_rate' = 0.0
-'ocean.models' = 'constant'
diff --git a/pism_terra/data/dem_ak.txt b/pism_terra/data/dem_ak.txt
new file mode 100644
index 0000000..becf6bf
--- /dev/null
+++ b/pism_terra/data/dem_ak.txt
@@ -0,0 +1,10 @@
+# QGIS Generated Color Map Export File
+INTERPOLATION:INTERPOLATED
+-2000,35,105,140,255,-2000.0000
+0,201,221,231,255,0
+1,230,230,230,255,1
+1250,153,153,153,255,1000
+2000,211,158,127,255,2000
+2500,89,36,40,255,2500
+3000,234,215,94,255,3000
+3500,191,191,191,255,3500
diff --git a/pism_terra/data/bath_topo.txt b/pism_terra/data/dem_gris.txt
similarity index 100%
rename from pism_terra/data/bath_topo.txt
rename to pism_terra/data/dem_gris.txt
diff --git a/pism_terra/data/ellsmere.gpkg b/pism_terra/data/ellsmere.gpkg
new file mode 100644
index 0000000..cadb873
Binary files /dev/null and b/pism_terra/data/ellsmere.gpkg differ
diff --git a/pism_terra/data/gris-basins-w-shelves.gpkg b/pism_terra/data/gris-basins-w-shelves.gpkg
new file mode 100644
index 0000000..7f7c09e
Binary files /dev/null and b/pism_terra/data/gris-basins-w-shelves.gpkg differ
diff --git a/pism_terra/data/postprocess_ismip7_scalar.sh b/pism_terra/data/postprocess_ismip7_scalar.sh
new file mode 100755
index 0000000..4837966
--- /dev/null
+++ b/pism_terra/data/postprocess_ismip7_scalar.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+# ISMIP7 post-processing script for scalar diagnostics
+#
+# Expects the file with scalar diagnostics to be named
+#
+# scalar_GrIS_UAF_PISM_XXX_XXX_XXX_XXX_XXX_YYYY-YYYY.nc, e.g.
+#
+# scalar_GrIS_UAF_PISM_m001_CESM2-WACCM_f001_historical_C001_1985-2014.nc
+#
+# when splitting this file, "scalar" will be replaced by the name of a scalar diagnostic.
+
+set -u
+set -e
+
+input=$1
+
+# Per-input scratch file (next to the input) so concurrent jobs don't collide on
+# a shared tmp.nc.
+tmp="${input%.nc}_pptmp.nc"
+
+# tell bash to remove the scratch file when done:
+trap 'rm -f "${tmp}"' EXIT
+
+# fix global attributes:
+ncatted -O \
+ -a crs,global,c,c,"EPSG:3413" \
+ -a command,global,d,c,"" \
+ -a source,global,d,c,"" \
+ ${input} "${tmp}"
+
+# fix time units:
+script='
+time=float(time/86400);
+time@units="days since 1850-01-01";
+time_bounds=float(time_bounds/86400);
+time_bounds@units="days since 1850-01-01"
+'
+
+ncap2 -O -s "${script}" "${tmp}" "${tmp}"
+
+snapshot_variables="
+lim
+limnsw
+iareagr
+iareafl
+"
+
+flux_variables="
+tendacabf
+tendlibmassbfgr
+tendlibmassbffl
+tendlicalvf
+tendlifmassbf
+tendligroundf
+"
+
+fill_value=9.9692099683868690e+36
+
+for var in ${snapshot_variables};
+do
+ output=${input/scalar/${var}}
+ # extract the variable
+ ncks -v ${var} -O "${tmp}" ${output}
+ # convert from double to float
+ ncap2 -s "${var}=float(${var})" -O ${output} ${output}
+ # set _FillValue
+ ncatted -a _FillValue,${var},c,f,${fill_value} -O ${output} ${output}
+done
+
+for var in ${flux_variables};
+do
+ output=${input/scalar/${var}}
+ # extract the variable
+ ncks -v ${var} -O "${tmp}" ${output}
+ # convert from double to float
+ ncap2 -s "${var}=float(${var})" -O ${output} ${output}
+ # set _FillValue and correct units
+ ncatted -a _FillValue,${var},c,f,${fill_value} \
+ -a units,${var},m,c,"kg s-1" \
+ -O ${output} ${output}
+ # correct the time dimension
+ ncap2 -s "time=float(0.5*(time_bounds(:,0)+time_bounds(:,1)))" -O ${output} ${output}
+done
diff --git a/pism_terra/data/speed_colorblind.txt b/pism_terra/data/speed.txt
similarity index 100%
rename from pism_terra/data/speed_colorblind.txt
rename to pism_terra/data/speed.txt
diff --git a/pism_terra/domain.py b/pism_terra/domain.py
index 0e57812..bb2b546 100644
--- a/pism_terra/domain.py
+++ b/pism_terra/domain.py
@@ -22,6 +22,7 @@
import geopandas as gpd
import numpy as np
+import shapely
import xarray as xr
@@ -83,10 +84,10 @@ def get_bounds(
ds : xarray.Dataset
The input dataset containing the x and y coordinates.
base_resolution : int, optional
- The base resolution in meters, by default 150.
+ The base resolution in meters, by default 50.
multipliers : list or numpy.ndarray, optional
A list or array of multipliers to compute the set of grid resolutions,
- by default [1, 2, 4].
+ by default [1, 2, 4, 5, 10, 20].
Returns
-------
@@ -120,6 +121,38 @@ def get_bounds(
return x_bnds, y_bnds
+def get_bounds_from_geometry(geom: shapely.geometry, buffer_dist: float = 2000.0, dx: float = 1000.0):
+ """
+ Compute a ``dx``-aligned bounding box around a buffered geometry.
+
+ The geometry is buffered by ``buffer_dist`` (in the geometry's CRS units),
+ then its bounds are snapped inward to a multiple of ``dx``.
+
+ Parameters
+ ----------
+ geom : shapely.geometry.base.BaseGeometry or geopandas.GeoSeries
+ Geometry (or GeoSeries) to buffer and bound. Must expose ``.buffer``
+ and a ``.bounds`` accessor with ``minx``/``maxx``/``miny``/``maxy``.
+ buffer_dist : float, default ``2000.0``
+ Buffer distance applied to the geometry, in CRS units (typically meters).
+ dx : float, default ``1000.0``
+ Grid spacing used to snap the bounds. ``x_min``/``y_min`` are rounded up
+ and ``x_max``/``y_max`` are rounded down to the nearest multiple of ``dx``.
+
+ Returns
+ -------
+ tuple of list of float
+ ``([x_min, x_max], [y_min, y_max])`` aligned to ``dx``.
+ """
+ bounds = geom.buffer(buffer_dist).bounds
+ x_min = np.ceil((bounds.minx.item()) / dx) * dx
+ x_max = np.floor((bounds.maxx.item()) / dx) * dx
+ y_min = np.ceil((bounds.miny.item()) / dx) * dx
+ y_max = np.floor((bounds.maxy.item()) / dx) * dx
+
+ return [x_min, x_max], [y_min, y_max]
+
+
def create_grid(
series: gpd.GeoSeries,
ds: xr.Dataset,
@@ -138,12 +171,12 @@ def create_grid(
ds : xarray.Dataset
The dataset containing the x and y coordinates.
buffer_distance : float, optional
- The buffer_distance distance around the geometry, by default 500.
+ The buffer_distance distance around the geometry, by default 1000.0.
base_resolution : int, optional
- The base resolution in meters, by default 150.
+ The base resolution in meters, by default 50.
multipliers : list or numpy.ndarray, optional
A list or array of multipliers to compute the set of grid resolutions,
- by default [1, 2, 4].
+ by default [1, 2, 4, 5, 8, 10, 20].
crs : str, optional
The coordinate reference system (CRS) for the domain, by default "EPSG:3413".
@@ -271,9 +304,7 @@ def create_domain(
data=0,
dims=[y_dim, x_dim],
coords={x_dim: coords[x_dim], y_dim: coords[y_dim]},
- attrs={
- "dimensions": f"{x_dim} {y_dim}",
- },
+ attrs={"dimensions": f"{x_dim} {y_dim}"},
),
x_bnds_dim: xr.DataArray(
data=x_bounds,
@@ -288,8 +319,16 @@ def create_domain(
},
attrs={"Conventions": "CF-1.8"},
).rio.set_spatial_dims(x_dim=x_dim, y_dim=y_dim)
- ds.rio.write_crs(crs, inplace=True).rio.write_coordinate_system(inplace=True)
+ # Use rioxarray's default grid-mapping variable name ("spatial_ref"). A
+ # non-default name here leaks downstream: derived datasets (e.g. the boot
+ # file) inherit this scalar coord, then add their own "spatial_ref" via
+ # write_crs, ending up with two grid-mapping variables. On write the stale
+ # one lands in the CF `coordinates` attr while `grid_mapping` points at the
+ # other, so on reload the CRS-bearing variable is not promoted to a coord
+ # and tools (rioxarray, QGIS) fail to detect the CRS.
+ ds = ds.rio.write_crs(crs).rio.write_coordinate_system()
+
for var in list(ds.data_vars) + list(ds.coords):
- ds[var].encoding["_FillValue"] = None
+ ds[var].encoding.update({"_FillValue": None})
return ds
diff --git a/pism_terra/download.py b/pism_terra/download.py
index 251f59f..b28e6a8 100644
--- a/pism_terra/download.py
+++ b/pism_terra/download.py
@@ -22,28 +22,35 @@
from __future__ import annotations
+import logging
import os
import re
import tarfile
import tempfile
+import time
import zipfile
from collections.abc import Iterable, Sequence
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
from pathlib import Path
from typing import NamedTuple
from urllib.parse import urlparse
import boto3
-import cdsapi
import earthaccess
import numpy as np
import requests
import xarray as xr
+from ecmwf.datastores import Client as _DatastoresClient
+from ecmwf.datastores import Remote as _DatastoresRemote
from tqdm.auto import tqdm
from pism_terra.aws import download_from_s3
from pism_terra.workflow import check_xr_lazy
+# Silence noisy third-party INFO chatter that interleaves with tqdm bars.
+for _name in ("cdsapi", "datapi", "multiurl", "ecmwf", "botocore", "s3transfer", "boto3"):
+ logging.getLogger(_name).setLevel(logging.WARNING)
+
class FileInfo(NamedTuple):
"""
@@ -259,125 +266,420 @@ def extract_archive(
return extracted_files
+def _cds_year_cache_path(dataset: str, request: dict, year: str, dest: Path, suffix: str = ".nc") -> Path:
+ """
+ Build a collision-resistant cache filename for a per-year CDS download.
+
+ Parameters
+ ----------
+ dataset : str
+ CDS dataset identifier (used in the filename).
+ request : dict
+ CDS request dict; only ``request["variable"]`` is read, to avoid
+ cache collisions when different variables are requested for the
+ same dataset/year.
+ year : str
+ Four-digit year string.
+ dest : Path
+ Directory the cache file lives in.
+ suffix : str, default ``".nc"``
+ File extension for the cached download (including the leading dot).
+ Use ``".grib"`` when ``request["format"]`` (or ``"data_format"``) is
+ ``"grib"`` so the on-disk extension matches the actual content.
+
+ Returns
+ -------
+ Path
+ Path of the form ``.nc`` (where + :func:`pism_terra.glacier.prepare` uploads them). **kwargs - E.g. force_overwrite. + E.g. ``force_overwrite`` (bool), ``bucket`` (str). Returns ------- - list[pathlib.Path] - Paths to the three 30-year climatology NetCDF files: - ``snap_1920_1949.nc``, ``snap_1950_1979.nc``, ``snap_1980_2009.nc``. - Additionally, the full stack ``snap_1900_2015.nc`` is written in - ``path`` as a side effect. + list of pathlib.Path + One forcing file per window: ``snap__
_.nc``.
"""
force_overwrite: bool = bool(kwargs.pop("force_overwrite", False))
+ bucket: str = str(kwargs.pop("bucket", "pism-cloud-data"))
+ _ = years # SNAP emits all windows; ``years`` only satisfies the dispatch signature.
+
+ print("")
+ print("Generate historical climate")
+ print("-" * 120)
- bucket: str = "pism-cloud-data"
out_dir = Path(path)
- for sn in ("snap_cru_TS40_1920_1949.nc", "snap_cru_TS40_1950_1979.nc", "snap_cru_TS40_1980_2009.nc"):
+ # Destination CRS/extent from the target grid; SNAP is on EPSG:3338, so clip
+ # in that CRS (PISM reprojects 3338 -> model grid via the grid mapping).
+ mapping_var = target_grid.rio.grid_mapping
+ dst_crs = target_grid[mapping_var].attrs["crs_wkt"]
+ bounds = [
+ target_grid.x_bnds.values[0][0],
+ target_grid.y_bnds.values[0][0],
+ target_grid.x_bnds.values[-1][-1],
+ target_grid.y_bnds.values[-1][-1],
+ ]
+ t = Transformer.from_crs(dst_crs, "EPSG:3338", always_xy=True)
+ minx, miny, maxx, maxy = t.transform_bounds(*bounds)
+
+ windows = {
+ (1920, 1949): "snap_cru_TS40_1920_1949.nc",
+ (1950, 1979): "snap_cru_TS40_1950_1979.nc",
+ (1980, 2009): "snap_cru_TS40_1980_2009.nc",
+ }
+ fs = s3fs.S3FileSystem(anon=True)
- snap_file = out_dir / sn
+ out_files: list[Path] = []
+ for (lo, hi), sn in windows.items():
+ snap_filename = out_dir / Path(f"snap_{lo}_{hi}_{rgi_id}.nc")
+ if check_xr_lazy(snap_filename) and not force_overwrite:
+ out_files.append(snap_filename)
+ continue
+ # Fetch the climatology from the same ``/climate`` location that
+ # prepare.py uploads to (matching stage.carra2()).
+ snap_file = out_dir / sn
if (not check_xr_lazy(snap_file)) or force_overwrite:
+ uri = f"s3://{bucket}/{prefix}/climate/{sn}".replace("//", "/").replace("s3:/", "s3://")
snap_file.unlink(missing_ok=True)
- s3_to_local(bucket, prefix="snap", dest=path)
- snap_files = list(Path(path).rglob("snap_*.nc"))
- return snap_files
+ fs.get(uri, str(snap_file))
+
+ ds = xr.open_dataset(snap_file, decode_coords="all")
+ ds = ds.rio.write_crs("EPSG:3338")
+ # Pad the clip by a few cells so PISM's reprojected domain stays a subset.
+ pad = 3 * float(abs(ds.x.values[1] - ds.x.values[0]))
+ ds = ds.rio.clip_box(minx - pad, miny - pad, maxx + pad, maxy + pad)
+ # SNAP rasters are north-up (descending y); PISM requires a strictly
+ # increasing y axis, so flip to ascending (and keep x ascending).
+ ds = ds.sortby(["y", "x"])
+
+ # Convert to PISM/ERA5 conventions (units PISM's atmosphere.given expects).
+ # ``air_temp_sd`` is an interannual standard deviation, so it stays a
+ # difference (no +273.15 offset). ``precipitation`` already arrives as a
+ # daily rate (kg m^-2 day^-1) from prepare_snap.
+ ds["air_temp"] = ds["air_temp"] + 273.15
+ ds["air_temp"].attrs.update({"units": "kelvin", "standard_name": "air_temperature"})
+ if "air_temp_sd" in ds:
+ ds["air_temp_sd"].attrs.update({"units": "kelvin"})
+ ds["precipitation"].attrs.update({"units": "kg m^-2 day^-1"})
+ if "surface" in ds:
+ ds["surface"].attrs.update({"units": "m", "standard_name": "surface_altitude"})
+
+ # Rebuild a CF monthly time axis + bounds so PISM reads the 12-month
+ # climatology as a periodic monthly cycle (see kitp.forcing.process_carra2).
+ # clip_box can drop ``time_bounds``, so re-create it deterministically.
+ if ds.sizes.get("time") == 12:
+ ds = ds.drop_vars(["time_bounds", "time_bnds"], errors="ignore")
+ month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+ bounds_start = np.cumsum([0] + month_lengths[:-1]).astype("float64")
+ bounds_end = np.cumsum(month_lengths).astype("float64")
+ time_mid = (bounds_start + bounds_end) / 2.0
+ time_bounds = np.column_stack([bounds_start, bounds_end])
+ ds = ds.assign_coords(time=("time", time_mid))
+ ds["time"].encoding.clear() # drop stale decode units so attrs win on write
+ ds["time"].attrs.update(
+ {
+ "standard_name": "time",
+ "axis": "T",
+ "units": "days since 0001-01-01",
+ "calendar": "365_day",
+ "bounds": "time_bounds",
+ }
+ )
+ ds["time_bounds"] = (("time", "nv"), time_bounds)
+
+ ds = _finalize_pism_crs(ds, "EPSG:3338")
+ drop_geotransform_attr(ds)
+
+ encoding = {
+ v: {"_FillValue": None}
+ for v in ("x", "y", "surface", "air_temp", "air_temp_sd", "precipitation", "time", "time_bounds")
+ if v in ds
+ }
+ # A per-variable encoding dict replaces the variable's ``.encoding``, so
+ # carry the CF ``grid_mapping`` set by _finalize_pism_crs through it.
+ for v in ds.data_vars:
+ grid_mapping = ds[v].encoding.get("grid_mapping")
+ if grid_mapping and v in encoding:
+ encoding[v]["grid_mapping"] = grid_mapping
+ snap_filename.unlink(missing_ok=True)
+ ds.to_netcdf(snap_filename, encoding=encoding, engine="h5netcdf", unlimited_dims=["time"])
+ out_files.append(snap_filename)
+
+ return out_files
+
+
+def _carra2_fill_years_and_bounds(ds: xr.Dataset, years: Sequence[int]) -> xr.Dataset:
+ """
+ Expand CARRA2 monthly data over ``years`` and attach monthly ``time_bnds``.
+
+ CARRA2 is only downloaded for a sparse set of source years (see
+ :func:`prepare_carra2`). For any target year not in the source, the 12
+ months of the *nearest* available source year are copied and re-stamped
+ with the target year. Ties are broken toward the earlier year (e.g.
+ 2004 → 2003). Bounds for each monthly timestamp are written as
+ ``[t, next-month-start)`` so PISM can interpret the data as monthly means.
+
+ Parameters
+ ----------
+ ds : xarray.Dataset
+ CARRA2 dataset with a monthly ``time`` coordinate (12 entries per
+ available year).
+ years : sequence of int
+ Target years to materialize in the output.
+
+ Returns
+ -------
+ xarray.Dataset
+ Same variables as ``ds``, with ``time`` expanded over ``years`` and a
+ new ``time_bnds`` variable. Any incoming ``time_bnds`` is rebuilt.
+ """
+
+ def _year(t):
+ """
+ Return the calendar year of ``t``.
+
+ Parameters
+ ----------
+ t : object
+ A scalar time value (cftime datetime, pandas Timestamp, or
+ ``numpy.datetime64``).
+
+ Returns
+ -------
+ int
+ Calendar year extracted from ``t``.
+ """
+ return t.year if hasattr(t, "year") else pd.Timestamp(t).year
+
+ def _replace_year(t, new_year):
+ """
+ Return ``t`` with its year set to ``new_year``.
+
+ Parameters
+ ----------
+ t : object
+ Scalar time value (cftime datetime, pandas Timestamp, or
+ ``numpy.datetime64``).
+ new_year : int
+ Year to assign on the returned value.
+
+ Returns
+ -------
+ object
+ Same dtype family as ``t`` (cftime in, cftime out;
+ ``numpy.datetime64`` in, ``numpy.datetime64`` out).
+ """
+ if hasattr(t, "replace") and not isinstance(t, np.datetime64):
+ return t.replace(year=new_year)
+ return np.datetime64(pd.Timestamp(t).replace(year=new_year))
+
+ def _next_month(t):
+ """
+ Return the first instant of the calendar month following ``t``.
+
+ Parameters
+ ----------
+ t : object
+ Scalar time value (cftime datetime, pandas Timestamp, or
+ ``numpy.datetime64``).
+
+ Returns
+ -------
+ object
+ ``t`` advanced to the first day of the next month, preserving the
+ input dtype family.
+ """
+ if isinstance(t, np.datetime64) or not hasattr(t, "month"):
+ ts = pd.Timestamp(t)
+ if ts.month == 12:
+ return np.datetime64(ts.replace(year=ts.year + 1, month=1))
+ return np.datetime64(ts.replace(month=ts.month + 1))
+ if t.month == 12:
+ return t.replace(year=t.year + 1, month=1)
+ return t.replace(month=t.month + 1)
+
+ ds = ds.drop_vars("time_bnds", errors="ignore")
+ src_times = ds["time"].values
+ src_year_of = np.array([_year(t) for t in src_times])
+ source_years = sorted(set(src_year_of.tolist()))
+
+ pieces = []
+ for ty in sorted({int(y) for y in years}):
+ nearest = min((abs(sy - ty), sy) for sy in source_years)[1]
+ sub = ds.isel(time=np.where(src_year_of == nearest)[0])
+ new_times = np.array([_replace_year(t, ty) for t in sub["time"].values])
+ pieces.append(sub.assign_coords(time=new_times))
+
+ merged = xr.concat(pieces, dim="time")
+
+ # Orography is time-invariant; concat (or an upstream broadcast) gives it a
+ # redundant time dimension with identical slices. Drop it back to a single
+ # 2-D field.
+ if "orography" in merged and "time" in merged["orography"].dims:
+ merged["orography"] = merged["orography"].isel(time=0, drop=True)
+
+ times = merged["time"].values
+ bounds = np.stack([times, np.array([_next_month(t) for t in times])], axis=1)
+ merged["time_bnds"] = xr.DataArray(bounds, dims=["time", "nv"], coords={"time": merged["time"]})
+ # CF time-axis identity so ncview/PISM recognise the time coordinate. (The
+ # time dimension must also be written unlimited — see the to_netcdf calls.)
+ merged["time"].attrs.update(
+ {
+ "standard_name": "time",
+ "long_name": "time",
+ "axis": "T",
+ "bounds": "time_bnds",
+ }
+ )
+ return merged
+
+
+def carra2(
+ target_grid: xr.Dataset,
+ rgi_id: str,
+ years: list[int] | Iterable[int] = range(1986, 2026),
+ path: Path | str = ".",
+ bucket: str = "pism-cloud-data",
+ prefix: str = "",
+ force_overwrite: bool = False,
+) -> Path:
+ """
+ Subset and reproject CARRA2 reanalysis to a glacier's target grid.
+
+ Opens the cloud-hosted CARRA2 Zarr store on S3, lazily clips it to the
+ bounding box of ``target_grid`` (after transforming the box into CARRA2
+ coordinates), reprojects the subset onto ``target_grid``, and writes a
+ compressed NetCDF.
+
+ Parameters
+ ----------
+ target_grid : xarray.Dataset
+ Target grid dataset providing the destination CRS (via ``spatial_ref``)
+ and extent. Used to derive the bounding box for the CARRA2 subset.
+ rgi_id : str
+ Glacier identifier, e.g., ``"RGI2000-v7.0-C-01-10853"``. Used in the
+ output filename.
+ years : list of int or Iterable of int, default ``range(1978, 2026)``
+ Years to materialize in the output. CARRA2 only stores a sparse set
+ of source years; any requested year not in the source is filled by
+ copying the nearest available source year (ties go to the earlier
+ year).
+ path : str or pathlib.Path, default ``"."``
+ Output directory. The function writes
+ ``carra2_.nc`` inside this directory.
+ bucket : str, default ``"pism-cloud-data"``
+ S3 bucket hosting the CARRA2 Zarr store.
+ prefix : str, default ``""``
+ Optional S3 key prefix; the full URI becomes
+ ``s3:////climate/carra2.zarr``.
+ force_overwrite : bool, default ``False``
+ If True, regenerate the output even if the cached NetCDF exists.
+
+ Returns
+ -------
+ pathlib.Path
+ Absolute path to the written NetCDF file.
+
+ Raises
+ ------
+ FileNotFoundError
+ If the Zarr store cannot be opened.
+ ValueError
+ If ``target_grid`` lacks a CRS or the bbox cannot be transformed.
+
+ Notes
+ -----
+ - Output variables and their units are inherited from the source CARRA2
+ Zarr store (typically ``air_temp`` in K, ``precipitation`` in
+ kg m^-2 day^-1, and dimensionless ``albedo``).
+ - Floating-point variables have NaNs filled with 0 for PISM; for
+ ``albedo`` this only affects polar-night months (no insolation), where
+ the value is irrelevant.
+ - Compression: zlib level 2 + shuffle.
+ """
+ path = Path(path)
+ path.mkdir(parents=True, exist_ok=True)
+
+ print("")
+ print("Generate historical climate")
+ print("-" * 120)
+
+ carra2_filename = path / Path(f"carra2_{rgi_id}.nc")
+ if carra2_filename.exists() and not force_overwrite and check_xr_lazy(carra2_filename):
+ print(f"Using cached {carra2_filename}")
+ return carra2_filename
+
+ # Bounding box of the target grid in its own (projected) CRS.
+ bounds = (
+ float(target_grid.x_bnds.values[0][0]),
+ float(target_grid.y_bnds.values[0][0]),
+ float(target_grid.x_bnds.values[-1][-1]),
+ float(target_grid.y_bnds.values[-1][-1]),
+ )
+ mapping_var = target_grid.rio.grid_mapping
+ dst_crs = target_grid[mapping_var].attrs["crs_wkt"]
+
+ # Fast path: prepare.py pre-reprojects CARRA2 once per S4F aggregate group
+ # and uploads ``carra2_.nc`` (CARRA2 ~2.5 km, already in the
+ # group's CRS). If that file exists on S3, fetch it and let PISM handle
+ # interpolation onto the model grid at runtime.
+ pre_key = f"{prefix}/climate/carra2_{rgi_id}.nc".lstrip("/")
+ pre_uri = f"s3://{bucket}/{pre_key}"
+ fs = s3fs.S3FileSystem(anon=True)
+ if fs.exists(pre_uri):
+ print(f"Found precomputed {pre_uri}; downloading")
+ tmp_pre = path / f"_carra2_pre_{rgi_id}.nc"
+ fs.get(pre_uri, str(tmp_pre))
+ with xr.open_dataset(tmp_pre) as pre:
+ out = _carra2_fill_years_and_bounds(pre.load(), list(years))
+ for v in out.data_vars:
+ if np.issubdtype(out[v].dtype, np.floating):
+ out[v] = out[v].fillna(0)
+ for name in list(out.coords) + list(out.data_vars):
+ for k in (
+ "dtype",
+ "_FillValue",
+ "units",
+ "calendar",
+ "chunks",
+ "preferred_chunks",
+ ):
+ out[name].encoding.pop(k, None)
+ encoding: dict[str, dict[str, object]] = {
+ name: {"zlib": True, "complevel": 2, "shuffle": True} for name in out.data_vars
+ }
+ encoding.update(
+ {
+ "time": {"dtype": "int64", "units": "hours since 1978-01-01 00:00:00"},
+ "time_bnds": {
+ "dtype": "int64",
+ "units": "hours since 1978-01-01 00:00:00",
+ },
+ }
+ )
+ # Re-stamp the grid mapping (dropped by the fill/fillna above) so PISM
+ # can find the projection and reproject the forcing on the fly.
+ out = _finalize_pism_crs(out, dst_crs)
+ carra2_filename.unlink(missing_ok=True)
+ out.to_netcdf(carra2_filename, encoding=encoding, engine="h5netcdf", unlimited_dims=["time"])
+ tmp_pre.unlink(missing_ok=True)
+ return carra2_filename
+
+ uri = f"s3://{bucket}/{prefix}/climate/carra2.zarr".replace("//", "/").replace("s3:/", "s3://")
+ print(f"Opening {uri}")
+ ds = xr.open_zarr(
+ uri,
+ consolidated=True,
+ storage_options={"anon": True},
+ chunks={},
+ )
+ print(f" opened zarr: vars={list(ds.data_vars)}, dims={dict(ds.sizes)}", flush=True)
+
+ # Make sure rioxarray knows which dims are spatial and what the CRS is.
+ # inplace=True avoids deep-copying the full lazy zarr just to stamp metadata.
+ if "x" in ds.dims and "y" in ds.dims:
+ ds.rio.set_spatial_dims(x_dim="x", y_dim="y", inplace=True)
+ elif "lon" in ds.dims and "lat" in ds.dims:
+ ds.rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=True)
+
+ # Recover CRS from whichever grid-mapping variable the Zarr writer used.
+ # CARRA2 stores it as a *data variable* (not a coord) named "crs"; other
+ # writers use "spatial_ref", "polar_stereographic", "lambert_conformal_conic", etc.
+ if ds.rio.crs is None:
+ crs_wkt = None
+ # First, follow any data variable's CF grid_mapping pointer.
+ for var in ds.data_vars:
+ gm = ds[var].attrs.get("grid_mapping") or ds[var].encoding.get("grid_mapping")
+ if gm and gm in ds.variables: # checks both data_vars and coords
+ crs_wkt = ds[gm].attrs.get("crs_wkt") or ds[gm].attrs.get("spatial_ref")
+ if crs_wkt:
+ break
+ # Fall back to scanning every variable for a CRS-shaped attr.
+ if not crs_wkt:
+ for name in ds.variables:
+ attrs = ds[name].attrs
+ crs_wkt = attrs.get("crs_wkt") or attrs.get("spatial_ref")
+ if crs_wkt:
+ break
+ if not crs_wkt:
+ raise ValueError(
+ f"Could not recover a CRS from the CARRA2 Zarr store at {uri}. "
+ f"Variables present: {list(ds.variables)}"
+ )
+ # inplace=True avoids a deep copy of the full lazy zarr.
+ ds.rio.write_crs(crs_wkt, inplace=True)
+ print(f" CRS resolved: {ds.rio.crs}", flush=True)
+
+ # Transform the target bbox into CARRA2 coordinates and clip there.
+ # Use .sel(x=slice, y=slice) instead of rio.clip_box because the latter
+ # tries to apply spatial dims to every Dataset variable and trips over
+ # non-spatial helpers like ``time_bnds``.
+ t = Transformer.from_crs(dst_crs, ds.rio.crs, always_xy=True)
+ minx, miny, maxx, maxy = t.transform_bounds(*bounds)
+ x_ascending = bool(ds.x[-1] > ds.x[0])
+ y_ascending = bool(ds.y[-1] > ds.y[0])
+ sub = ds.sel(
+ x=slice(minx, maxx) if x_ascending else slice(maxx, minx),
+ y=slice(miny, maxy) if y_ascending else slice(maxy, miny),
+ )
+ print(f" subset dims: {dict(sub.sizes)}", flush=True)
+
+ # rioxarray.reproject_match eagerly reads every non-spatial coord (via
+ # ``coord.values``). The published CARRA2 Zarr has at least one coord
+ # (e.g. time_bnds) whose stored chunk bytes don't match its declared
+ # dtype, so the lazy read aborts with a numpy-view error. Pre-load each
+ # non-spatial coord; drop the ones that don't survive.
+ #
+ # The grid-mapping placeholder (typically named ``crs`` or
+ # ``spatial_ref``) is dropped unconditionally — we already recovered the
+ # CRS into a clean rioxarray ``spatial_ref`` above and don't need the
+ # original 0-d variable's data.
+ grid_mapping_names = set()
+ for var in sub.data_vars:
+ gm = sub[var].attrs.get("grid_mapping") or sub[var].encoding.get("grid_mapping")
+ if gm:
+ grid_mapping_names.add(gm)
+ for c in list(sub.coords):
+ if c in ("x", "y", "spatial_ref"):
+ continue
+ if c in grid_mapping_names:
+ sub = sub.drop_vars(c, errors="ignore")
+ continue
+ try:
+ sub = sub.assign_coords({c: sub[c].compute()})
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ print(f"Coord {c!r} unreadable from Zarr ({exc}); dropping")
+ sub = sub.drop_vars(c, errors="ignore")
+ # If `time.bounds` pointed at a coord we just dropped, clear the attr too.
+ if "time" in sub.coords:
+ bounds_name = sub["time"].attrs.get("bounds")
+ if bounds_name and bounds_name not in sub.coords and bounds_name not in sub.data_vars:
+ sub["time"].attrs.pop("bounds", None)
+
+ sub = sub.drop_vars("time_bnds", errors="ignore")
+
+ # Reproject the subset onto the target grid. Each reprojected batch is
+ # written to disk immediately so we don't hoard them all in RAM — for
+ # large aggregates the target grid can be enormous (100 m × 100s of km)
+ # and a single batch already costs many GB. We then assemble the per-
+ # variable results lazily from disk and merge into `out`.
+ #
+ # Suppress any registered dask Callback (e.g. ProgressBar from an outer
+ # scope) so the [#####] 100% lines don't interleave with the tqdm bar.
+ import tempfile # pylint: disable=import-outside-toplevel
+
+ from dask.callbacks import Callback # pylint: disable=import-outside-toplevel
+
+ # Size the per-batch reprojected buffer to ~200 MB. Each batch holds
+ # time_batch × target_y × target_x float32s in RAM after rasterio.warp,
+ # so the cap matters most for high-res, wide-bbox runs like S4F aggregates.
+ target_cells = int(target_grid.sizes["x"]) * int(target_grid.sizes["y"])
+ time_batch = max(1, min(24, 200_000_000 // (target_cells * 4)))
+ print(f" target_grid: {dict(target_grid.sizes)}, time_batch={time_batch}", flush=True)
+ out_vars: dict[str, xr.DataArray] = {}
+ saved_callbacks = Callback.active.copy()
+ Callback.active.clear()
+ # Track every shard DataArray we lazy-open so we can close them
+ # explicitly before the TemporaryDirectory exit tries to unlink them.
+ # Without the explicit close, non-deterministic GC leaves file handles
+ # open and NFS / networked filesystems emit ``.nfs*`` lock files that
+ # then trip ``rmtree`` with ``ENOTEMPTY`` on the per-variable subdirs.
+ open_shards: list[xr.DataArray] = []
+ try:
+ with tempfile.TemporaryDirectory(prefix="carra2_reproj_", dir=str(path)) as tmp_dir_name:
+ tmp_dir = Path(tmp_dir_name)
+ for var_name in sub.data_vars:
+ da = sub[var_name]
+ print(f" processing {var_name} dims={dict(da.sizes)}", flush=True)
+ if "time" not in da.dims or da.sizes["time"] <= time_batch:
+ out_vars[var_name] = da.rio.reproject_match(target_grid, resampling=Resampling.bilinear).astype(
+ "float32"
+ )
+ continue
+ n = da.sizes["time"]
+ var_dir = tmp_dir / var_name
+ var_dir.mkdir(parents=True, exist_ok=True)
+ batch_files: list[Path] = []
+ pbar = tqdm(
+ range(0, n, time_batch),
+ desc=f"Reprojecting {var_name}",
+ unit="batch",
+ leave=False,
+ )
+ for batch_idx, i in enumerate(pbar):
+ chunk = da.isel(time=slice(i, i + time_batch)).compute()
+ reproj = chunk.rio.reproject_match(target_grid, resampling=Resampling.bilinear).astype("float32")
+ batch_path = var_dir / f"batch_{batch_idx:04d}.nc"
+ # Clear inherited encoding before writing each shard.
+ reproj.encoding = {}
+ reproj.to_netcdf(batch_path, engine="h5netcdf")
+ batch_files.append(batch_path)
+ # Drop the in-memory copy so the next batch starts clean.
+ del chunk, reproj
+ pbar.close()
+ # Lazy assemble: open each shard chunked, concat along time.
+ shards = [xr.open_dataarray(p, chunks={}) for p in batch_files]
+ open_shards.extend(shards)
+ out_vars[var_name] = xr.concat(shards, dim="time")
+ out = xr.Dataset(out_vars, attrs=sub.attrs)
+
+ # Stamp CRS metadata while everything is still lazy. inplace=True
+ # avoids deep-copying a multi-GiB dataset just to attach attrs.
+ out = (
+ out.rio.write_crs(dst_crs, inplace=True)
+ .rio.write_grid_mapping(inplace=True)
+ .rio.write_coordinate_system(inplace=True)
+ )
+ out.attrs["Conventions"] = "CF-1.8"
+
+ # Expand to all requested years (filling missing ones from the
+ # nearest source year) and attach CF time_bnds for PISM.
+ out = _carra2_fill_years_and_bounds(out, list(years))
+ # Per-variable NaN fills. 0 is physically fine for precipitation
+ # and orography (no precip / sea level at the masked corners),
+ # but plugging 0 into ``air_temp`` gives 0 K, which is
+ # unphysical and blows up PISM's energy balance. Use 260 K
+ # (well below any real surface temperature we'd care about
+ # preserving) so the corners have a plausible cold value.
+ # ``air_temp_sd`` stays at 0 because a missing standard
+ # deviation is best interpreted as "no variability observed".
+ fill_defaults: dict[str, float] = {
+ "air_temp": 260.0,
+ "air_temp_sd": 0.0,
+ "precipitation": 0.0,
+ "orography": 0.0,
+ }
+ for v in out.data_vars:
+ if not np.issubdtype(out[v].dtype, np.floating):
+ continue
+ fill_value = fill_defaults.get(str(v), 0.0)
+ out[v] = out[v].fillna(fill_value)
+
+ # Clear stale encoding inherited from the Zarr source so netCDF4
+ # doesn't see half-prescribed datetime encoding (e.g. dtype=int64
+ # with no units).
+ for name in list(out.coords) + list(out.data_vars):
+ for k in (
+ "dtype",
+ "_FillValue",
+ "units",
+ "calendar",
+ "chunks",
+ "preferred_chunks",
+ ):
+ out[name].encoding.pop(k, None)
+
+ # Compressed NetCDF (zlib level 2 + shuffle for floats).
+ encoding_c: dict[str, dict[str, object]] = {
+ name: {"zlib": True, "complevel": 2, "shuffle": True} for name in out.data_vars
+ }
+ encoding_c.update(
+ {
+ "time": {"dtype": "int64", "units": "hours since 1978-01-01 00:00:00"},
+ "time_bnds": {
+ "dtype": "int64",
+ "units": "hours since 1978-01-01 00:00:00",
+ },
+ }
+ )
+
+ # Re-stamp the grid mapping (dropped by the concat/fillna above) so
+ # PISM can find the projection and reproject the forcing on the fly.
+ out = _finalize_pism_crs(out, dst_crs)
+
+ # Stream from the per-batch shards straight into the final NetCDF
+ # — never materializing the full reprojected dataset in RAM, which
+ # for high-res grids over 50 years is easily 10+ GiB per variable.
+ # Writing here (inside the temp-dir context) keeps the shards alive
+ # for dask to read from.
+ carra2_filename.unlink(missing_ok=True)
+ out.to_netcdf(carra2_filename, encoding=encoding_c, engine="h5netcdf", unlimited_dims=["time"])
+
+ # Release every file handle *before* the TemporaryDirectory
+ # context manager runs its rmtree — otherwise networked-FS
+ # ``.nfs*`` lock files strand the per-variable subdirs and
+ # cleanup dies with ``ENOTEMPTY``.
+ out.close()
+ for shard in open_shards:
+ shard.close()
+ open_shards.clear()
+ out_vars.clear()
+ del out
+ gc.collect()
+ finally:
+ Callback.active.update(saved_callbacks)
+
+ return carra2_filename
def era5(
+ target_grid: xr.Dataset,
rgi_id: str,
- rgi: gpd.GeoDataFrame | str | Path = "rgi/rgi.gpkg",
- years: list[int] | Iterable[int] = range(1978, 2025),
+ years: list[int] | Iterable[int] = range(1978, 2026),
dataset: str = "reanalysis-era5-land-monthly-means",
- buffer_distance: float = 0.1,
path: Path | str = ".",
**kwargs,
) -> Path:
"""
Download monthly ERA5 reanalysis over a glacier bounding box and write a NetCDF.
- Given a glacier ``rgi_id``, this function:
- (1) loads the glacier geometry (GeoDataFrame or GPKG),
- (2) builds a buffered lon/lat bounding box (WGS84),
- (3) requests ERA5 monthly means (2 m air temperature and total precipitation),
- (4) optionally fills missing values using a global product,
- (5) adds a representative geopotential (converted to meters),
- (6) writes a CF-compliant NetCDF under ``path`` and returns its absolute path.
-
Parameters
----------
+ target_grid : xarray.Dataset
+ Target grid dataset providing the destination CRS (via ``spatial_ref``)
+ and extent. Used to derive the geographic bounding box for the ERA5
+ request.
rgi_id : str
- Glacier identifier, e.g., ``"RGI2000-v7.0-C-01-10853"``.
- rgi : geopandas.GeoDataFrame or str or pathlib.Path, default ``"rgi/rgi.gpkg"``
- In-memory RGI table or a path to a GeoPackage readable by
- :func:`geopandas.read_file`.
- years : list of int or Iterable of int, default ``range(1980, 2025)``
+ Glacier identifier, e.g., ``"RGI2000-v7.0-C-01-10853"``. Used in the
+ output filename.
+ years : list of int or Iterable of int, default ``range(1978, 2026)``
Years to request from ERA5.
- dataset : str, default ``"reanalysis-era5-single-levels-monthly-means"``
+ dataset : str, default ``"reanalysis-era5-land-monthly-means"``
CDS dataset name for monthly single-level means (ERA5). Adjust if you
intend to query ERA5-Land or other products.
- buffer_distance : float, default ``0.1``
- Buffer (degrees) applied to the glacier footprint before subsetting.
- path : str or pathlib.Path, default ``"era5.nc"``
+ path : str or pathlib.Path, default ``"."``
Output directory or filename base. The function writes a file named
``era5_wgs84_.nc`` inside ``path`` if ``path`` is a directory;
otherwise the provided filename is used.
@@ -367,23 +1814,22 @@ def era5(
print("")
print("Generate historical climate")
- print("-" * 80)
+ print("-" * 120)
era5_filename = path / Path(f"era5_wgs84_{rgi_id}.nc")
- if isinstance(rgi, (str, Path)):
- rgi = gpd.read_file(rgi)
-
years = list(years)
- glacier = get_glacier_from_rgi_id(rgi, rgi_id).to_crs("EPSG:4326")
- minx, miny, maxx, maxy = glacier.iloc[0]["geometry"].buffer(buffer_distance).bounds
- area = [
- np.ceil(maxy * 10) / 10,
- np.floor(minx * 10) / 10,
- np.floor(miny * 10) / 10,
- np.ceil(maxx * 10) / 10,
+ bounds = [
+ target_grid.x_bnds.values[0][0],
+ target_grid.y_bnds.values[0][0],
+ target_grid.x_bnds.values[-1][-1],
+ target_grid.y_bnds.values[-1][-1],
]
+ mapping_var = target_grid.rio.grid_mapping
+ dst_crs = target_grid[mapping_var].attrs["crs_wkt"]
+ t = Transformer.from_crs(dst_crs, "EPSG:4326")
+ area = t.transform_bounds(*bounds)
print(f"Bounding box {area}")
@@ -395,11 +1841,22 @@ def era5(
era5_filename_2 = path / Path(f"era5_wgs84_{rgi_id}_tmp_2.nc")
era5_files.append(era5_filename_2)
ds_geo = (
- download_request(dataset, area, [2013], variable=["geopotential"], file_path=era5_filename_2, **kwargs)
- .squeeze()
+ download_request(
+ dataset,
+ area,
+ [2013],
+ variable=["geopotential"],
+ file_path=era5_filename_2,
+ **kwargs,
+ )
+ .squeeze("time", drop=True)
.drop_vars("time", errors="ignore")
)
- ds_geo_ = ds_geo.rio.write_crs("EPSG:4326").rio.reproject_match(ds).rename({"x": "longitude", "y": "latitude"})
+ ds_geo_ = (
+ ds_geo.rio.write_crs("EPSG:4326")
+ .rio.reproject_match(ds, resampling=Resampling.bilinear)
+ .rename({"x": "longitude", "y": "latitude"})
+ )
lon_attrs = ds["longitude"].attrs
lat_attrs = ds["latitude"].attrs
@@ -409,14 +1866,22 @@ def era5(
era5_filename_3 = path / Path(f"era5_wgs84_{rgi_id}_tmp_3.nc")
era5_files.append(era5_filename_3)
ds_global = download_request(
- "reanalysis-era5-single-levels-monthly-means", area, years, file_path=era5_filename_3, **kwargs
+ "reanalysis-era5-single-levels-monthly-means",
+ area,
+ years,
+ file_path=era5_filename_3,
+ **kwargs,
)
ds_global_ = (
- ds_global.rio.write_crs("EPSG:4326").rio.reproject_match(ds).rename({"x": "longitude", "y": "latitude"})
+ ds_global.rio.write_crs("EPSG:4326")
+ .rio.reproject_match(ds, resampling=Resampling.bilinear)
+ .rename({"x": "longitude", "y": "latitude"})
)
- ds = xr.where(np.isnan(ds), ds_global_, ds)
+ common_vars = list(set(ds.data_vars) & set(ds_global_.data_vars))
+ for v in common_vars:
+ ds[v] = xr.where(np.isnan(ds[v]), ds_global_[v], ds[v])
- ds = xr.merge([ds, ds_geo_], combine_attrs="override")
+ ds = xr.merge([ds, ds_geo_], compat="no_conflicts")
ds = ds.rename({"valid_time": "time"})
ds = ds.rename_vars({"tp": "precipitation", "t2m": "air_temp", "z": "surface"})
@@ -440,6 +1905,182 @@ def era5(
return era5_filename
+def era5_mean(
+ target_grid: xr.Dataset,
+ rgi_id: str,
+ years: list[int] | Iterable[int] = range(1990, 2020),
+ dataset: str = "reanalysis-era5-land-monthly-means",
+ path: Path | str = ".",
+ **kwargs,
+) -> Path:
+ """
+ Download monthly ERA5 reanalysis over a glacier bounding box and write a NetCDF.
+
+ Parameters
+ ----------
+ target_grid : xarray.Dataset
+ Target grid dataset providing the destination CRS (via ``spatial_ref``)
+ and extent. Used to derive the geographic bounding box for the ERA5
+ request.
+ rgi_id : str
+ Glacier identifier, e.g., ``"RGI2000-v7.0-C-01-10853"``. Used in the
+ output filename.
+ years : list of int or Iterable of int, default ``range(1978, 2026)``
+ Years to request from ERA5.
+ dataset : str, default ``"reanalysis-era5-land-monthly-means"``
+ CDS dataset name for monthly single-level means (ERA5). Adjust if you
+ intend to query ERA5-Land or other products.
+ path : str or pathlib.Path, default ``"."``
+ Output directory or filename base. The function writes a file named
+ ``era5_wgs84_.nc`` inside ``path`` if ``path`` is a directory;
+ otherwise the provided filename is used.
+ **kwargs
+ Additional keyword arguments forwarded to :func:`download_request`
+ (e.g., alternate ``variable`` sequences, custom authentication/session
+ options, or client settings). These are passed unchanged to the CDS
+ retrieval helper.
+
+ Returns
+ -------
+ pathlib.Path
+ Absolute path to the written NetCDF file.
+
+ Raises
+ ------
+ FileNotFoundError
+ If the provided RGI path is missing.
+ ValueError
+ If the glacier ID cannot be found or the geometry is invalid.
+ Exception
+ Any errors propagated from the CDS request, reprojection, or I/O.
+
+ See Also
+ --------
+ download_request
+ Helper that performs the CDS API query and returns an xarray object.
+ geopandas.read_file
+ Load the RGI vector layer from disk.
+ xarray.Dataset.rio.write_crs
+ Record CRS on xarray objects via rioxarray.
+
+ Notes
+ -----
+ - Output variables:
+ - ``air_temp`` (K) from ERA5 ``t2m``.
+ - ``precipitation`` (kg m^-2 day^-1) from ERA5 ``tp`` (converted).
+ - ``surface`` (m) derived from ERA5 ``z`` / 9.80665 (geopotential → meters).
+ - ``time_bounds`` are added for CF-style climatological metadata.
+ - If missing values are detected in the regional subset, the function
+ patches them from the global reanalysis (same period).
+ """
+ path = Path(path)
+
+ print("")
+ print("Generate historical climate")
+ print("-" * 120)
+
+ years = range(1990, 2020)
+
+ era5_filename = path / Path(f"era5_wgs84_{rgi_id}.nc")
+
+ bounds = [
+ target_grid.x_bnds.values[0][0],
+ target_grid.y_bnds.values[0][0],
+ target_grid.x_bnds.values[-1][-1],
+ target_grid.y_bnds.values[-1][-1],
+ ]
+ mapping_var = target_grid.rio.grid_mapping
+ dst_crs = target_grid[mapping_var].attrs["crs_wkt"]
+ t = Transformer.from_crs(dst_crs, "EPSG:4326")
+ area = t.transform_bounds(*bounds)
+
+ print(f"Bounding box {area}")
+
+ era5_files = []
+ era5_filename_1 = path / Path(f"era5_wgs84_{rgi_id}_tmp_1.nc")
+ era5_files.append(era5_filename_1)
+ ds = download_request(dataset, area, years, file_path=era5_filename_1, **kwargs)
+
+ era5_filename_2 = path / Path(f"era5_wgs84_{rgi_id}_tmp_2.nc")
+ era5_files.append(era5_filename_2)
+ ds_geo = (
+ download_request(
+ dataset,
+ area,
+ [2013],
+ variable=["geopotential"],
+ file_path=era5_filename_2,
+ **kwargs,
+ )
+ .squeeze("time", drop=True)
+ .drop_vars("time", errors="ignore")
+ )
+ ds_geo_ = (
+ ds_geo.rio.write_crs("EPSG:4326")
+ .rio.reproject_match(ds, resampling=Resampling.bilinear)
+ .rename({"x": "longitude", "y": "latitude"})
+ )
+
+ lon_attrs = ds["longitude"].attrs
+ lat_attrs = ds["latitude"].attrs
+
+ if bool(ds.to_array().isnull().any().item()):
+ print("Missing values detected, filling with global reanalysis")
+ era5_filename_3 = path / Path(f"era5_wgs84_{rgi_id}_tmp_3.nc")
+ era5_files.append(era5_filename_3)
+ ds_global = download_request(
+ "reanalysis-era5-single-levels-monthly-means",
+ area,
+ years,
+ file_path=era5_filename_3,
+ **kwargs,
+ )
+ ds_global_ = (
+ ds_global.rio.write_crs("EPSG:4326")
+ .rio.reproject_match(ds, resampling=Resampling.bilinear)
+ .rename({"x": "longitude", "y": "latitude"})
+ )
+ common_vars = list(set(ds.data_vars) & set(ds_global_.data_vars))
+ for v in common_vars:
+ ds[v] = xr.where(np.isnan(ds[v]), ds_global_[v], ds[v])
+
+ ds = xr.merge([ds, ds_geo_], compat="no_conflicts")
+ # Time-mean snapshot, but preserve a length-1 ``time`` dim anchored at the
+ # midpoint of the source range. ERA5 monthly stamps are first-of-month, so
+ # the climatology covers ``[min(src), max(src) + 1 month)``.
+ renamed = ds.rename({"valid_time": "time"})
+ src_times = renamed["time"].values
+ midpoint = renamed["time"].mean().values
+ src_lo = src_times.min()
+ src_hi = (pd.Timestamp(src_times.max()) + pd.offsets.MonthBegin(1)).to_datetime64()
+ ds = renamed.mean(dim="time", keep_attrs=True).expand_dims(time=[midpoint])
+ ds["air_temp_sd"] = renamed["t2m"].std(dim="time", keep_attrs=True).expand_dims(time=[midpoint])
+
+ ds = ds.rename_vars({"tp": "precipitation", "t2m": "air_temp", "z": "surface"})
+ ds["surface"] /= 9.80665
+ ds["surface"].attrs.update({"units": "m", "standard_name": "surface_altitude"})
+ ds["precipitation"] *= 1000
+ ds["precipitation"].attrs.update({"units": "kg m^-2 day^-1"})
+ ds["air_temp"].attrs.update({"units": "kelvin"})
+ ds["air_temp_sd"].attrs.update({"units": "kelvin"})
+ ds["time"].encoding["units"] = "hours since 1980-01-01 00:00:00"
+ ds["time"].encoding["calendar"] = "standard"
+ ds["longitude"].attrs = lon_attrs
+ ds["latitude"].attrs = lat_attrs
+ ds.rio.write_crs("EPSG:4326", inplace=True)
+ for name in ("latitude", "longitude", "surface", "precipitation", "air_temp"):
+ if name in ds:
+ ds[name].encoding.update({"_FillValue": None})
+
+ # Length-1 climatology: write bounds spanning the source range directly
+ # (add_time_bounds builds N-1 pairs and would empty a single-step series).
+ ds["time_bounds"] = xr.DataArray(np.array([[src_lo, src_hi]]), dims=("time", "nv"))
+ ds["time"].attrs["bounds"] = "time_bounds"
+ ds.to_netcdf(era5_filename)
+
+ return era5_filename
+
+
def jif_cosipy(url: str, download_path: Path | str, path: Path | str) -> None:
"""
Download and prepare COSIPY.
@@ -452,11 +2093,6 @@ def jif_cosipy(url: str, download_path: Path | str, path: Path | str) -> None:
The path to the original file.
path : str, Path
The path to the processed file.
-
- Returns
- -------
- xr.Dataset
- An xarray Dataset containing COSIPY.
"""
if Path(download_path).exists():
@@ -498,139 +2134,189 @@ def jif_cosipy(url: str, download_path: Path | str, path: Path | str) -> None:
ds.to_netcdf(path)
-def pmip4(
+def era5_monthly_mean(
+ target_grid: xr.Dataset,
rgi_id: str,
- rgi: gpd.GeoDataFrame | str | Path = "rgi/rgi.gpkg",
- buffer_distance: float = 2.0,
+ years: list[int] | Iterable[int] = range(1978, 2026),
+ dataset: str = "reanalysis-era5-land-monthly-means",
path: Path | str = ".",
**kwargs,
-) -> list[Path]:
+) -> Path:
"""
- Build PMIP4 LGM monthly climatology over a glacier bbox and write one NetCDF per model.
-
- For the glacier identified by ``rgi_id``, this function:
- (1) reads the glacier geometry (GeoDataFrame or GPKG) and converts to WGS84,
- (2) constructs a buffered lon/lat bounding box (degrees),
- (3) pulls PMIP4/CMIP6 LGM monthly fields (``tas``, ``pr``) from the
- ``pangeo-cmip6`` S3 catalog,
- (4) subsets to the bounding box, merges variables, and selects the final 2,400 months,
- (5) computes a 12-month climatology (groupby month → mean),
- (6) writes one CF-style NetCDF per ``source_id`` into ````.
+ Download monthly ERA5 reanalysis over a glacier bounding box and write a NetCDF.
Parameters
----------
+ target_grid : xarray.Dataset
+ Target grid dataset providing the destination CRS (via ``spatial_ref``)
+ and extent. Used to derive the geographic bounding box for the ERA5
+ request.
rgi_id : str
- RGI glacier identifier (e.g., ``"RGI2000-v7.0-C-01-10853"``).
- rgi : geopandas.GeoDataFrame or str or pathlib.Path, default ``"rgi/rgi.gpkg"``
- Either an in-memory RGI GeoDataFrame, or a path to a GeoPackage readable by
- :func:`geopandas.read_file`.
- buffer_distance : float, default ``2.0``
- Buffer (degrees) applied to the glacier footprint before subsetting.
+ Glacier identifier, e.g., ``"RGI2000-v7.0-C-01-10853"``. Used in the
+ output filename.
+ years : list of int or Iterable of int, default ``range(1978, 2026)``
+ Years to request from ERA5.
+ dataset : str, default ``"reanalysis-era5-land-monthly-means"``
+ CDS dataset name for monthly single-level means (ERA5). Adjust if you
+ intend to query ERA5-Land or other products.
path : str or pathlib.Path, default ``"."``
- Output directory where files are written as
- ``{source_id}_rgi_id_{rgi_id}.nc``.
+ Output directory or filename base. The function writes a file named
+ ``era5_wgs84_.nc`` inside ``path`` if ``path`` is a directory;
+ otherwise the provided filename is used.
**kwargs
- Reserved for future options (e.g., catalog filtering). Currently unused.
+ Additional keyword arguments forwarded to :func:`download_request`
+ (e.g., alternate ``variable`` sequences, custom authentication/session
+ options, or client settings). These are passed unchanged to the CDS
+ retrieval helper.
Returns
-------
- list of pathlib.Path
- Absolute paths to the written NetCDF files, one per CMIP6 ``source_id``.
+ pathlib.Path
+ Absolute path to the written NetCDF file.
Raises
------
FileNotFoundError
- If the RGI path is missing (when ``rgi`` is a path).
+ If the provided RGI path is missing.
ValueError
- If the glacier ID cannot be found or geometry/CRS is invalid.
+ If the glacier ID cannot be found or the geometry is invalid.
Exception
- Errors propagated from S3 access (``s3fs``), zarr decoding, or file writing.
+ Any errors propagated from the CDS request, reprojection, or I/O.
+
+ See Also
+ --------
+ download_request
+ Helper that performs the CDS API query and returns an xarray object.
+ geopandas.read_file
+ Load the RGI vector layer from disk.
+ xarray.Dataset.rio.write_crs
+ Record CRS on xarray objects via rioxarray.
Notes
-----
- - Variables are renamed to:
- - ``air_temp`` (from ``tas``)
- - ``precipitation`` (from ``pr``)
- and the 12-month climatology dimension is renamed to ``time``.
- - The output CRS is set to ``EPSG:4326`` via rioxarray.
- - A CF-compliant 12-month ``time`` coordinate is attached using
- ``cftime.DatetimeNoLeap`` for a nominal year and encoded with
- ``calendar="365_day"`` and ``units="days since 0001-01-01"``. Time bounds
- (``time_bounds``) are added.
- - Longitudinal bounds are wrapped to 0–360 using modulo arithmetic; the
- bbox edges are rounded to tenths of a degree.
+ - Output variables:
+ - ``air_temp`` (K) from ERA5 ``t2m``.
+ - ``precipitation`` (kg m^-2 day^-1) from ERA5 ``tp`` (converted).
+ - ``surface`` (m) derived from ERA5 ``z`` / 9.80665 (geopotential → meters).
+ - ``time_bounds`` are added for CF-style climatological metadata.
+ - If missing values are detected in the regional subset, the function
+ patches them from the global reanalysis (same period).
"""
+ path = Path(path)
print("")
- print("Generate PMIP4 LGM climate")
- print("-" * 80)
+ print("Generate historical climate")
+ print("-" * 120)
- force_overwrite: bool = bool(kwargs.pop("force_overwrite", False))
+ era5_filename = path / Path(f"era5_wgs84_{rgi_id}.nc")
- if isinstance(rgi, str | Path):
- rgi = gpd.read_file(rgi)
+ _ = years
+ years = [1978]
- glacier = get_glacier_from_rgi_id(rgi, rgi_id).to_crs("EPSG:4326")
- minx, miny, maxx, maxy = glacier.iloc[0]["geometry"].buffer(buffer_distance).bounds
+ bounds = [
+ target_grid.x_bnds.values[0][0],
+ target_grid.y_bnds.values[0][0],
+ target_grid.x_bnds.values[-1][-1],
+ target_grid.y_bnds.values[-1][-1],
+ ]
+ mapping_var = target_grid.rio.grid_mapping
+ dst_crs = target_grid[mapping_var].attrs["crs_wkt"]
+ t = Transformer.from_crs(dst_crs, "EPSG:4326")
+ area = t.transform_bounds(*bounds)
- minx = (np.floor(minx * 10) / 10) % 360
- maxx = (np.ceil(maxx * 10) / 10) % 360
- miny = np.floor(miny * 10) / 10
- maxy = np.ceil(maxy * 10) / 10
+ print(f"Bounding box {area}")
- print(f"Bounding box {minx}, {maxx}, {miny}, {maxy}")
+ era5_files = []
+ era5_filename_1 = path / Path(f"era5_wgs84_{rgi_id}_tmp_1.nc")
+ era5_files.append(era5_filename_1)
+ ds = download_request(dataset, area, years, file_path=era5_filename_1, **kwargs)
- fs = s3fs.S3FileSystem(anon=True)
- time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
- cmip6_df = pd.read_csv("https://cmip6-pds.s3.amazonaws.com/pangeo-cmip6.csv")
- lgm_df = cmip6_df.query(
- "activity_id=='PMIP' & table_id=='Amon' & experiment_id=='lgm' & (variable_id=='tas' | variable_id=='pr')"
+ era5_filename_2 = path / Path(f"era5_wgs84_{rgi_id}_tmp_2.nc")
+ era5_files.append(era5_filename_2)
+ ds_geo = (
+ download_request(
+ dataset,
+ area,
+ [2013],
+ variable=["geopotential"],
+ file_path=era5_filename_2,
+ **kwargs,
+ )
+ .squeeze("time", drop=True)
+ .drop_vars("time", errors="ignore")
+ )
+ ds_geo_ = (
+ ds_geo.rio.write_crs("EPSG:4326")
+ .rio.reproject_match(ds, resampling=Resampling.bilinear)
+ .rename({"x": "longitude", "y": "latitude"})
)
- responses = []
- for source_id, df in lgm_df.groupby(by="source_id"):
- path = Path(path)
- p = path / f"{source_id}_rgi_id_{rgi_id}.nc"
-
- if (not check_xr_fully(p)) or force_overwrite:
- dss = []
- for v in ["tas", "pr"]:
- zstore = df[df["variable_id"] == v].zstore.values[0]
- mapper = fs.get_mapper(zstore)
-
- # open using xarray
- ds = (
- xr.open_zarr(mapper, consolidated=True, decode_times=time_coder, decode_timedelta=True).drop_vars(
- ["height"], errors="ignore"
- )
- ).sel({"lon": slice(minx, maxx), "lat": slice(miny, maxy)})
-
- dss.append(ds)
- ds = (
- xr.merge(dss)
- .isel({"time": slice(-2401, -1)})
- .groupby("time.month")
- .mean()
- .rename_dims({"month": "time"})
- .rename_vars({"pr": "precipitation", "tas": "air_temp", "month": "time"})
- )
- ds.rio.write_crs("EPSG:4326", inplace=True)
+ lon_attrs = ds["longitude"].attrs
+ lat_attrs = ds["latitude"].attrs
- month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
- bounds_start = np.cumsum([0] + month_lengths[:-1]).astype("float64")
- bounds_end = np.cumsum(month_lengths).astype("float64")
- time_mid = (bounds_start + bounds_end) / 2.0
+ if bool(ds.to_array().isnull().any().item()):
+ print("Missing values detected, filling with global reanalysis")
+ era5_filename_3 = path / Path(f"era5_wgs84_{rgi_id}_tmp_3.nc")
+ era5_files.append(era5_filename_3)
+ ds_global = download_request(
+ "reanalysis-era5-single-levels-monthly-means",
+ area,
+ years,
+ file_path=era5_filename_3,
+ **kwargs,
+ )
+ ds_global_ = (
+ ds_global.rio.write_crs("EPSG:4326")
+ .rio.reproject_match(ds, resampling=Resampling.bilinear)
+ .rename({"x": "longitude", "y": "latitude"})
+ )
+ common_vars = list(set(ds.data_vars) & set(ds_global_.data_vars))
+ for v in common_vars:
+ ds[v] = xr.where(np.isnan(ds[v]), ds_global_[v], ds[v])
+
+ ds = xr.merge([ds, ds_geo_], compat="no_conflicts")
+ # Time-mean snapshot, but preserve a length-1 ``time`` dim anchored at the
+ # midpoint of the source range. ERA5 monthly stamps are first-of-month, so
+ # the climatology covers ``[min(src), max(src) + 1 month)``.
+ ds = ds.rename({"valid_time": "time"})
- time_bounds = np.column_stack([bounds_start, bounds_end])
+ ds = ds.rename_vars({"tp": "precipitation", "t2m": "air_temp", "z": "surface"})
+ ds["surface"] /= 9.80665
+ ds["surface"].attrs.update({"units": "m", "standard_name": "surface_altitude"})
+ ds["precipitation"] *= 1000
+ ds["precipitation"].attrs.update({"units": "kg m^-2 day^-1"})
+ ds["air_temp"].attrs.update({"units": "kelvin"})
+ ds["longitude"].attrs = lon_attrs
+ ds["latitude"].attrs = lat_attrs
- ds = ds.assign_coords(time=("time", time_mid))
- ds["time"].attrs.update({"units": "days since 0001-01-01", "calendar": "365_day", "bounds": "time_bounds"})
- ds["time_bounds"] = (("time", "nv"), time_bounds)
+ ds = ds.drop_vars("time_bnds", errors="ignore")
+
+ month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+ bounds_start = np.cumsum([0] + month_lengths[:-1]).astype("float64")
+ bounds_end = np.cumsum(month_lengths).astype("float64")
+ time_mid = (bounds_start + bounds_end) / 2.0
+
+ time_bounds = np.column_stack([bounds_start, bounds_end])
+
+ ds = ds.assign_coords(time=("time", time_mid))
+ ds["time"].attrs.update(
+ {
+ "standard_name": "time",
+ "units": "days since 0001-01-01",
+ "calendar": "365_day",
+ "bounds": "time_bounds",
+ }
+ )
+ ds["time_bounds"] = (("time", "nv"), time_bounds)
- ds.to_netcdf(p)
+ ds.rio.write_crs("EPSG:4326", inplace=True)
+ for name in ("latitude", "longitude", "surface", "precipitation", "air_temp"):
+ if name in ds:
+ ds[name].encoding.update({"_FillValue": None})
- responses.append(p)
- return responses
+ ds.to_netcdf(era5_filename)
+
+ return era5_filename
def prepare_snap(
@@ -658,9 +2344,8 @@ def prepare_snap(
-------
list[pathlib.Path]
Paths to the three 30-year climatology NetCDF files:
- ``snap_1920_1949.nc``, ``snap_1950_1979.nc``, ``snap_1980_2009.nc``.
- Additionally, the full stack ``snap_1900_2015.nc`` is written in
- ``path`` as a side effect.
+ ``snap_cru_TS40_1920_1949.nc``, ``snap_cru_TS40_1950_1979.nc``,
+ ``snap_cru_TS40_1980_2009.nc``.
Raises
------
@@ -687,9 +2372,9 @@ def prepare_snap(
Examples
--------
- >>> out_paths = prepare_snap("RGI2000-v7.0-C-01-04374", path="snap_outputs", force_overwrite=True)
+ >>> out_paths = prepare_snap(path="snap_outputs", force_overwrite=True)
>>> [p.name for p in out_paths]
- ['snap_1920_1949.nc', 'snap_1950_1979.nc', 'snap_1980_2009.nc']
+ ['snap_cru_TS40_1920_1949.nc', 'snap_cru_TS40_1950_1979.nc', 'snap_cru_TS40_1980_2009.nc']
"""
print("")
@@ -732,9 +2417,15 @@ def prepare_snap(
ds = xr.merge(dss).rio.write_crs("EPSG:3338").fillna(0)
ds = ds.rename_vars({"pr": "precipitation", "tas": "air_temp"})
- ds["precipitation"] *= 12
- ds["precipitation"].attrs.update({"units": "kg m^-2 year^-1"})
ds["air_temp"].attrs.update({"units": "celsius"})
+ # ``precipitation`` is the monthly total (kg m^-2); it is converted to a
+ # daily rate per calendar month inside the period loop.
+
+ # Fixed-length months for the 365_day calendar used by the climatologies.
+ month_lengths = np.array([31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31], dtype=float)
+ month_edges = np.concatenate([[0.0], np.cumsum(month_lengths)]) # 13 edges, days
+ time_mid = month_edges[:-1] + month_lengths / 2.0
+ time_bounds = np.column_stack([month_edges[:-1], month_edges[1:]])
period_starts = [1920, 1950, 1980]
ps: list[Path] = []
@@ -751,15 +2442,22 @@ def prepare_snap(
# Reuse existing file; no work scheduled
continue
- # --- compute weighted monthly climatology for this period (all lazy) ---
- ds_sub = ds.sel(time=slice(start, end))
- month_length = ds_sub.time.dt.days_in_month
- weights = month_length.groupby("time.month") / month_length.groupby("time.month").sum()
-
- ds_weighted = (ds_sub * weights).groupby("time.month").sum(dim="time").rename({"month": "time"})
+ # --- compute the 12-month climatology for this period (all lazy) ---
+ # Mean over years for each calendar month; ``surface`` (no time dim)
+ # passes through unchanged and stays time-invariant.
+ ds_sel = ds.sel(time=slice(start, end))
+ ds_weighted = ds_sel.groupby("time.month").mean("time").rename({"month": "time"})
+ ds_weighted["air_temp_sd"] = ds_sel["air_temp"].groupby("time.month").std("time").rename({"month": "time"})
+ # The surface DEM is time-invariant; groupby broadcasts it across the 12
+ # month groups, so collapse the redundant time dimension back to 2-D.
+ if "surface" in ds_weighted and "time" in ds_weighted["surface"].dims:
+ ds_weighted["surface"] = ds_weighted["surface"].isel(time=0, drop=True)
# coordinate metadata on x/y
- for c, axis, stdname in (("x", "X", "projection_x_coordinate"), ("y", "Y", "projection_y_coordinate")):
+ for c, axis, stdname in (
+ ("x", "X", "projection_x_coordinate"),
+ ("y", "Y", "projection_y_coordinate"),
+ ):
attrs = {
"units": "m",
"axis": axis,
@@ -769,19 +2467,26 @@ def prepare_snap(
if c in ds_weighted.coords:
ds_weighted[c].attrs.update(attrs)
- month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
- bounds_start = np.cumsum([0] + month_lengths[:-1]).astype("float64")
- bounds_end = np.cumsum(month_lengths).astype("float64")
- time_mid = (bounds_start + bounds_end) / 2.0
-
- time_bounds = np.column_stack([bounds_start, bounds_end])
-
+ # Monthly mid-point time axis (days, 365_day calendar) with month bounds.
ds_weighted = ds_weighted.assign_coords(time=("time", time_mid))
ds_weighted["time"].attrs.update(
- {"units": "days since 0001-01-01", "calendar": "365_day", "bounds": "time_bounds"}
+ {
+ "standard_name": "time",
+ "axis": "T",
+ "units": "days since 0001-01-01",
+ "calendar": "365_day",
+ "bounds": "time_bounds",
+ }
)
ds_weighted["time_bounds"] = (("time", "nv"), time_bounds)
+ # Convert each month's precipitation total (kg m^-2) to a daily rate.
+ ml = xr.DataArray(month_lengths, coords={"time": ds_weighted["time"]}, dims="time")
+ ds_weighted["precipitation"] = ds_weighted["precipitation"] / ml
+ ds_weighted["precipitation"].attrs.update({"units": "kg m^-2 day^-1"})
+ ds_weighted["air_temp"].attrs.update({"units": "celsius"})
+ ds_weighted["air_temp_sd"].attrs.update({"units": "celsius"})
+
# CRS + spatial dims + grid_mapping tags
ds_weighted = ds_weighted.rio.set_spatial_dims(x_dim="x", y_dim="y")
ds_weighted.rio.write_crs("EPSG:3338", inplace=True)
@@ -792,13 +2497,22 @@ def prepare_snap(
# encoding: remove _FillValue without nuking other encodings
encoding = {
v: {"_FillValue": None}
- for v in ("x", "y", "surface", "air_temp", "precipitation", "time", "time_bounds")
+ for v in (
+ "x",
+ "y",
+ "surface",
+ "air_temp",
+ "air_temp_sd",
+ "precipitation",
+ "time",
+ "time_bounds",
+ )
if v in ds_weighted
}
# schedule a lazy NetCDF write for this period
p.unlink(missing_ok=True)
- delayed_writes.append(ds_weighted.to_netcdf(p, encoding=encoding, compute=False))
+ delayed_writes.append(ds_weighted.to_netcdf(p, encoding=encoding, compute=False, engine="h5netcdf"))
# Kick off all writes in parallel (plus internal dask parallelism)
if delayed_writes:
diff --git a/pism_terra/glacier/dem.py b/pism_terra/glacier/dem.py
index 3339348..ed2371c 100644
--- a/pism_terra/glacier/dem.py
+++ b/pism_terra/glacier/dem.py
@@ -24,162 +24,141 @@
from __future__ import annotations
+import collections
from pathlib import Path
-from typing import Literal
+from typing import Literal, Sequence
-import geopandas as gpd
import numpy as np
-import rasterio
-import rioxarray as rxr
+import rioxarray as rxr # noqa: F401 -- registers the .rio accessor
import xarray as xr
from dem_stitcher import stitch_dem
+from pyproj import Transformer
+from rasterio.enums import Resampling
-from pism_terra.domain import create_domain
from pism_terra.glacier.ice_thickness import get_ice_thickness
-from pism_terra.glacier.observations import glacier_velocities_from_rgi_id
-from pism_terra.raster import reproject_file
-from pism_terra.vector import get_glacier_from_rgi_id
-from pism_terra.workflow import check_rio
+from pism_terra.glacier.observations import (
+ bathymetry_from_grid,
+ glacier_velocities_from_grid,
+)
+from pism_terra.workflow import check_xr_lazy, drop_geotransform_attr
xr.set_options(keep_attrs=True)
def get_surface_dem_by_bounds(
- bounds: tuple[float, float, float, float],
+ bounds: Sequence[float],
dataset: Literal["glo_30", "arcticdem"],
path: str | Path = "input",
force_overwrite: bool = False,
+ cache_name: str | None = None,
) -> Path:
"""
- Create (or reuse) a surface DEM GeoTIFF for a geographic bounding box.
+ Create (or reuse) a surface DEM NetCDF for a geographic bounding box.
- Mosaics/exports a DEM over ``bounds`` via :func:`stitch_dem`, writes a
- single-band GeoTIFF with appropriate tags, and returns the output path.
- If a file with the expected name already exists under ``path`` and opens
- successfully via :func:`check_rio`, that file is reused unless
- ``force_overwrite=True``.
+ Mosaics/exports a DEM over ``bounds`` via :func:`stitch_dem` and writes the
+ result as a CF-compliant NetCDF (``netcdf4`` engine) with the CRS encoded
+ via rioxarray's ``grid_mapping``. If a file with the expected name already
+ exists under ``path`` and opens successfully via :func:`check_xr_lazy`,
+ that file is reused unless ``force_overwrite=True``.
Parameters
----------
bounds : tuple of float
Geographic bounding box ``(minx, miny, maxx, maxy)`` in degrees (WGS84).
- dataset : str, default ``"glo_30"``
- DEM source identifier recognized by :func:`stitch_dem`
- (e.g., ``"glo_30"``, ``"arcticdem"``).
+ dataset : {"glo_30", "arcticdem"}
+ DEM source identifier recognized by :func:`stitch_dem`.
path : str or pathlib.Path, default ``"input"``
- Output directory for the GeoTIFF. Created if it does not exist.
+ Output directory for the NetCDF cache. Created if it does not exist.
force_overwrite : bool, default ``False``
- If ``True``, skip cache reuse and regenerate the DEM even if a readable
- file already exists at the target location.
+ If ``True``, regenerate the DEM even if a readable cache already exists.
+ cache_name : str or None, optional
+ Stem to use for the NetCDF cache filename instead of ``dataset``.
+ Lets callers materialize multiple bbox slices of the same source
+ side by side (e.g. ``"glo_30_west"`` and ``"glo_30_east"`` for an
+ antimeridian-crossing region). Default is ``None`` (use ``dataset``).
Returns
-------
pathlib.Path
- Path to the GeoTIFF file containing the DEM.
-
- Raises
- ------
- FileNotFoundError
- If required DEM tiles cannot be fetched/assembled.
- ValueError
- If the stitched DEM/profile is invalid or incompatible with GeoTIFF.
- rasterio.errors.RasterioIOError
- On errors writing the GeoTIFF.
- Exception
- Any other error propagated by :func:`stitch_dem` or I/O routines.
-
- See Also
- --------
- stitch_dem
- Assemble a DEM mosaic and return the array and raster profile.
- check_rio
- Lightweight validity check for raster files readable by rioxarray.
-
- Notes
- -----
- - Output is a **single-band** GeoTIFF tagged with ``AREA_OR_POINT="Point"``.
- - Heights follow the profile from :func:`stitch_dem`
- (here ``dst_ellipsoidal_height=False`` typically implies orthometric heights).
- - The file is **not** deleted automatically; callers manage lifecycle.
-
- Examples
- --------
- >>> tif = get_surface_dem_by_bounds((214.1, 59.0, 219.7, 63.9),
- ... dem_name="glo_30", path="input")
- >>> tif.exists()
- True
+ Path to the NetCDF cache containing the DEM
+ (``.nc`` under ``path``).
"""
out_dir = Path(path)
out_dir.mkdir(parents=True, exist_ok=True)
- geoid_file = out_dir / f"{dataset}.tif"
- # Reuse if present and readable
- if (not check_rio(geoid_file)) or force_overwrite:
- X, p = stitch_dem(
- bounds,
- dem_name=dataset,
- dst_ellipsoidal_height=False,
- dst_area_or_point="Point",
- )
+ geo_file = out_dir / f"{cache_name or dataset}.nc"
- # Ensure the profile matches what we're writing
- p = p.copy()
- p.update(
- {
- "driver": "GTiff",
- "count": 1,
- "dtype": X.dtype, # e.g., 'float32'
- "BIGTIFF": "YES", # allow files >4 GB
- }
- )
+ if check_xr_lazy(geo_file, verbose=False) and not force_overwrite:
+ return geo_file
+
+ print(bounds)
+ X, p = stitch_dem(
+ bounds,
+ dem_name=dataset,
+ dst_ellipsoidal_height=False,
+ dst_area_or_point="Point",
+ )
- with rasterio.open(geoid_file, "w", **p) as src:
- src.write(X, 1) # write band 1
- src.update_tags(AREA_OR_POINT="Point")
+ transform = p["transform"]
+ height, width = X.shape
+ # Pixel-center coordinates derived from the affine transform.
+ xs = transform.c + transform.a * (np.arange(width) + 0.5)
+ ys = transform.f + transform.e * (np.arange(height) + 0.5)
+
+ da = xr.DataArray(
+ np.asarray(X, dtype="float32"),
+ dims=("y", "x"),
+ coords={"x": xs, "y": ys},
+ name="surface",
+ attrs={"AREA_OR_POINT": "Point"},
+ )
+ da = da.rio.write_crs(p["crs"]).rio.write_coordinate_system()
+ ds = da.to_dataset()
+ ds.attrs["Conventions"] = "CF-1.8"
- return geoid_file
+ geo_file.unlink(missing_ok=True)
+ # Drop the GeoTransform so GDAL/QGIS derive the (top-down) transform from the
+ # ascending y coordinate instead of rendering the raster upside-down.
+ drop_geotransform_attr(ds)
+ ds.to_netcdf(geo_file, engine="h5netcdf")
+ return geo_file
def prepare_surface(
- bounds: tuple[float, float, float, float],
- crs: str,
+ target_grid: xr.Dataset,
dataset: Literal["glo_30", "arcticdem"],
path: str | Path = "input_files",
- resolution: float = 50.0,
**kwargs,
-) -> tuple[Path, Path]:
+) -> xr.DataArray:
"""
- Prepare a surface DEM and matching target grid over a glacier extent.
+ Prepare a surface DEM aligned to a target grid.
Workflow:
- (1) Download/mosaic a DEM over ``bounds`` (geographic),
- (2) reproject/resample it to ``crs`` at ``resolution``,
- (3) crop to a 100 m–aligned box in projected coordinates,
- (4) generate a regular target grid covering that box, and
- (5) write both the surface (NetCDF) and the target grid (NetCDF) to ``path``.
+ (1) Derive a geographic (WGS84) bounding box from ``target_grid``.
+ (2) Download/mosaic a DEM over that bounding box.
+ (3) Reproject/resample it to match ``target_grid`` (CRS, extent, resolution).
+ (4) Write the result to ``surface.nc`` under ``path`` and return it.
Parameters
----------
- bounds : tuple of float
- Geographic bounding box as ``(minx, miny, maxx, maxy)`` in WGS84 degrees.
- crs : str
- Target coordinate reference system (e.g., ``"EPSG:32606"``).
- dataset : str, default ``"glo_30"``
+ target_grid : xarray.Dataset
+ Target grid dataset providing the destination CRS
+ and the cell-edge bounds (``x_bnds``/``y_bnds``) used to derive both the
+ geographic query bounds and the destination grid for reprojection.
+ dataset : {"glo_30", "arcticdem"}
DEM source identifier passed to :func:`get_surface_dem_by_bounds`.
path : str or pathlib.Path, default ``"input_files"``
Output directory for generated files. Created if missing.
- resolution : float, default ``50.0``
- Target grid spacing (meters) used during reprojection and grid creation.
**kwargs
Additional keyword arguments forwarded to :func:`get_surface_dem_by_bounds`
(e.g., ``force_overwrite=True``). These do not affect the reprojection
- or grid creation steps directly.
+ step directly.
Returns
-------
- tuple of pathlib.Path
- ``(surface_file, target_grid_file)``:
- - ``surface_file``: NetCDF with variable ``surface`` (meters).
- - ``target_grid_file``: NetCDF with the regular grid (coords/bounds).
+ xarray.DataArray
+ Surface elevation (meters) on ``target_grid``, with
+ ``standard_name="surface_altitude"``. Also written to
+ ``Path(path) / "surface.nc"``.
Raises
------
@@ -193,95 +172,106 @@ def prepare_surface(
Notes
-----
- The surface variable is named ``"surface"`` with
- ``standard_name="land_ice_elevation"`` and ``units="m"``.
- - Cropping aligns the domain to multiples of 100 m in projected x/y:
- ``x_min = ceil(min(x)/100)*100`` and ``x_max = floor(max(x)/100)*100``
- (similarly for y).
- - After interpolation to the target grid, missing values are filled with 0.
+ ``standard_name="surface_altitude"`` and ``units="m"``.
+ - After reprojection to the target grid, missing values are filled with 0.
Adjust if downstream tooling expects masked NaNs instead.
-
- Examples
- --------
- >>> surf_nc, grid_nc = prepare_surface(
- ... bounds=(214.1, 59.0, 219.7, 63.9),
- ... crs="EPSG:32606",
- ... dem_name="glo_30",
- ... path="input_files",
- ... resolution=50.0,
- ... force_overwrite=True, # forwarded via **kwargs
- ... )
"""
- geoid_file = get_surface_dem_by_bounds(bounds, dataset=dataset, path=path, **kwargs)
- projected_file = reproject_file(geoid_file, crs, resolution)
-
- surface = rxr.open_rasterio(projected_file).squeeze().drop_vars("band", errors="ignore")
- surface.name = "surface"
- surface.attrs.update({"standard_name": "land_ice_elevation", "units": "m"})
-
- # Round in projected meters to a 100 m grid
- x_min = np.ceil((surface.x.min()) / 100) * 100
- x_max = np.floor((surface.x.max()) / 100) * 100
- y_min = np.ceil((surface.y.min()) / 100) * 100
- y_max = np.floor((surface.y.max()) / 100) * 100
-
- target_grid = create_domain([x_min, x_max], [y_min, y_max], resolution=resolution, crs=crs)
-
- surface = surface.interp_like(target_grid).fillna(0)
+ bounds = [
+ target_grid.x_bnds.values[0][0],
+ target_grid.y_bnds.values[0][0],
+ target_grid.x_bnds.values[-1][-1],
+ target_grid.y_bnds.values[-1][-1],
+ ]
+ mapping_var = target_grid.rio.grid_mapping
+ dst_crs = target_grid[mapping_var].attrs["crs_wkt"]
+ t = Transformer.from_crs(dst_crs, "EPSG:4326", always_xy=True)
+ geo_bounds = t.transform_bounds(*bounds)
+
+ # pyproj signals an antimeridian-crossing 4326 bbox by returning
+ # xmin > xmax (e.g. RGI region 01 spans ~170°E..-117°W). dem_stitcher
+ # rejects that, so split into two halves at ±180°, stitch each, reproject
+ # each into the projected target grid (which is continuous across the
+ # antimeridian), then merge.
+ if geo_bounds[0] > geo_bounds[2]:
+ west_bounds = (geo_bounds[0], geo_bounds[1], 180.0, geo_bounds[3])
+ east_bounds = (-180.0, geo_bounds[1], geo_bounds[2], geo_bounds[3])
+ west_file = get_surface_dem_by_bounds(
+ west_bounds, dataset=dataset, path=path, cache_name=f"{dataset}_west", **kwargs
+ )
+ east_file = get_surface_dem_by_bounds(
+ east_bounds, dataset=dataset, path=path, cache_name=f"{dataset}_east", **kwargs
+ )
+ west = rxr.open_rasterio(west_file).squeeze().drop_vars("band", errors="ignore")
+ east = rxr.open_rasterio(east_file).squeeze().drop_vars("band", errors="ignore")
+ west_reproj = west.rio.reproject_match(target_grid, resampling=Resampling.bilinear)
+ east_reproj = east.rio.reproject_match(target_grid, resampling=Resampling.bilinear)
+ # Either half may cover any given target cell; coalesce, then fill the
+ # nodata gap at ±180° (one row of cells with no source pixel either
+ # side of the cut).
+ surface_reprojected = west_reproj.fillna(east_reproj).fillna(0)
+ surface_reprojected.name = "surface"
+ surface_reprojected.attrs.update({"standard_name": "surface_altitude", "units": "m"})
+ else:
+ geo_file = get_surface_dem_by_bounds(geo_bounds, dataset=dataset, path=path, **kwargs)
+ # rxr.open_rasterio reliably propagates CRS on a netCDF read;
+ # xr.open_dataset + manual write_crs misses grid_mapping unless the
+ # variable has the attribute.
+ surface = rxr.open_rasterio(geo_file).squeeze().drop_vars("band", errors="ignore")
+ surface.name = "surface"
+ surface.attrs.update({"standard_name": "surface_altitude", "units": "m"})
+ surface_reprojected = surface.rio.reproject_match(target_grid, resampling=Resampling.bilinear).fillna(0)
surface_file = Path(path) / "surface.nc"
- surface.to_netcdf(surface_file)
+ drop_geotransform_attr(surface_reprojected)
+ surface_reprojected.to_netcdf(surface_file)
- target_grid_file = Path(path) / "target_grid.nc"
- target_grid.to_netcdf(target_grid_file)
+ return surface_reprojected
- return surface_file, target_grid_file
-
-def boot_file_from_rgi_id(
+def boot_file_from_grid(
+ target_grid: xr.Dataset,
rgi_id: str,
- rgi: gpd.GeoDataFrame | str | Path,
+ geometries: collections.abc.Iterable,
dem_dataset: Literal["glo_30", "arcticdem"],
- ice_thickness_dataset: Literal["maffezzoli", "millan"],
+ ice_thickness_dataset: Literal["frank", "maffezzoli", "millan"],
+ bathymetry_dataset: Literal["none", "gebco"] | None,
velocity_dataset: Literal["none", "its_live"] | None,
- buffer_distance: float = 5000.0,
+ forcing_mask: Literal["none", "all", "glacier"] | None,
path: str | Path = "input_files",
- resolution: float = 50.0,
**kwargs,
) -> xr.Dataset:
"""
Build a glacier “boot” dataset (surface, thickness, bed, masks, aux vars) from an RGI ID.
- Steps:
- (1) Locate the glacier geometry by ``rgi_id`` (GeoDataFrame or file-based RGI).
- (2) Buffer the glacier polygon in its native projected CRS (meters).
- (3) Mosaic/reproject a DEM over the buffered extent at ``resolution``.
- (4) Interpolate glacier ice thickness onto the target grid.
- (5) Derive bed elevation and boolean masks from DEM clipping.
- (6) Optionally fetch observed velocities and create a simple ``tillwat`` field.
Returns a regular 2-D xarray Dataset in the glacier’s projected CRS.
Parameters
----------
+ target_grid : xarray.Dataset
+ Target grid dataset (with ``x``/``y`` coords, ``x_bnds``/``y_bnds`` cell-edge
+ bounds onto which all
+ derived fields are aligned.
rgi_id : str
Glacier identifier, e.g., ``"RGI2000-v7.0-C-06-00014"``.
- rgi : geopandas.GeoDataFrame or str or pathlib.Path, default ``"rgi/rgi.gpkg"``
- In-memory RGI table or a path to a GeoPackage/shape readable by
- :func:`geopandas.read_file`. Must contain a row with ``rgi_id`` and an
- ``epsg`` column specifying the glacier CRS.
- dem_dataset : str, default ``"glo_30"``
- DEM source for surface preparation (e.g., ``"glo_30"``, ``"arcticdem"``).
- ice_thickness_dataset : str, default ``"maffezzoli"``
- Source for ice thickness (e.g., ``"glo_30"``, ``"arcticdem"``).
- velocity_dataset : str, default ``"its_live"``
- Source for velocities (e.g., ``"none"``, ``"its_live"``).
- buffer_distance : float, default ``5000.0``
- Buffer distance **in meters** applied to the glacier polygon in the projected CRS
- to define the working extent for DEM/thickness.
+ geometries : iterable of shapely geometries
+ Glacier outline(s) in ``target_grid``'s CRS, used for clipping the DEM
+ and constructing masks.
+ dem_dataset : {"glo_30", "arcticdem"}
+ DEM source for surface preparation.
+ ice_thickness_dataset : {"frank", "maffezzoli", "millan"}
+ Source for ice thickness.
+ bathymetry_dataset : {"none", "gebco"} or None
+ Source for ocean bathymetry. When set, a cloud raster (e.g.
+ ``s3://...//bathymetry.tif``) is fetched, clipped
+ to the target grid, and used to fill ``bed`` where ``surface <= 0``.
+ ``"none"`` or ``None`` disables bathymetry merging.
+ velocity_dataset : {"none", "its_live"} or None
+ Source for velocities.
+ forcing_mask : {"none", "glacier", "all"} or None
+ FTT mask.
path : str or pathlib.Path, default ``"input_files"``
Working directory used by helper routines to cache/write intermediate rasters/grids.
- resolution : float, default ``50.0``
- Target grid spacing (meters) for reprojection/resampling.
**kwargs
Forwarded to :func:`prepare_surface` (e.g., ``force_overwrite=True``) and any
downstream helpers it calls. Does not alter variable naming/semantics.
@@ -296,7 +286,7 @@ def boot_file_from_rgi_id(
- ``land_ice_area_fraction_retreat`` : bool — 1 where DEM is ice-free after clipping.
- ``ftt_mask`` : bool — complementary outside-footprint mask (1 outside).
- ``tillwat`` : float32, m — simple basal water proxy (here ``0`` or ``2`` m based on speed).
- - ``v`` and related velocity fields (from :func:`glacier_velocities_from_rgi_id`), reprojected
+ - ``v`` and related velocity fields (from :func:`glacier_velocities_from_grid`), reprojected
to the surface grid, if available.
CRS is recorded via the rioxarray accessor (``.rio.crs``); spatial dims are set with
@@ -319,81 +309,108 @@ def boot_file_from_rgi_id(
Mosaic/reproject a DEM over a geographic bounding box and build the target grid.
get_ice_thickness
Interpolate glacier ice thickness onto a target grid.
- create_domain
- Create a regular xarray grid with specified bounds and resolution.
- glacier_velocities_from_rgi_id
+ glacier_velocities_from_grid
Retrieve observed surface velocities for the glacier domain.
-
- Notes
- -----
- - Buffering is done in the glacier’s projected CRS (meters). The buffered geometry is
- converted to WGS84 only to derive geographic bounds for DEM staging.
- - ``land_ice_area_fraction_retreat`` and ``ftt_mask`` are derived from DEM clipping and
- are intentionally simple; refine as needed for your application.
- - ``tillwat`` is a coarse proxy here: set to ``2 m`` where reprojected speed ``v >= 100 m/yr``,
- else ``0 m``. Adjust the threshold and values per your physics.
- - The function returns an **in-memory** dataset; it does not write to disk.
"""
print("")
print("Generate DEM")
- print("-" * 80)
-
- # Accept GeoDataFrame or a path to the RGI layer
- if isinstance(rgi, (str, Path)):
- rgi = gpd.read_file(rgi)
+ print("-" * 120)
- glacier = get_glacier_from_rgi_id(rgi, rgi_id)
- glacier_series = glacier.iloc[0]
- dst_crs = glacier_series["epsg"]
+ bucket: str = kwargs.pop("bucket", "pism-cloud-data")
+ prefix: str = kwargs.pop("prefix", "rgi")
- glacier_projected = glacier.to_crs(dst_crs)
- geometry_buffered_projected = glacier_projected.geometry.buffer(buffer_distance)
- geometry_buffered_geoid = geometry_buffered_projected.to_crs("EPSG:4326").iloc[0]
+ mapping_var = target_grid.rio.grid_mapping
+ dst_crs = target_grid[mapping_var].attrs["crs_wkt"]
- bounds_geoid_buffered = geometry_buffered_geoid.bounds
- surface_file, target_grid_file = prepare_surface(
- bounds_geoid_buffered, dst_crs, dataset=dem_dataset, path=path, resolution=resolution
- )
- surface = xr.open_dataarray(surface_file)
+ surface = prepare_surface(target_grid, dataset=dem_dataset, path=path)
surface = surface.where(surface > 0.0, 0.0)
- target_grid = xr.open_dataset(target_grid_file)
ice_thickness = get_ice_thickness(
- glacier, dataset=ice_thickness_dataset, path=path, target_grid=target_grid, target_crs=dst_crs, **kwargs
+ rgi_id,
+ dataset=ice_thickness_dataset,
+ path=path,
+ target_grid=target_grid,
+ target_crs=dst_crs,
+ bucket=bucket,
+ prefix=prefix,
+ geometries=geometries,
+ **kwargs,
)
- ice_thickness = ice_thickness.rio.clip(glacier_projected.geometry, drop=False).fillna(0)
+ ice_thickness = ice_thickness.rio.reproject_match(target_grid, resampling=Resampling.bilinear)
+ ice_thickness = ice_thickness.rio.clip(geometries, drop=False).fillna(0)
ice_thickness = ice_thickness.where(ice_thickness > 0.0, 0.0)
bed = surface - ice_thickness
- bed.name = "bed"
+ if bathymetry_dataset not in ("none", None):
+ bathymetry_uri = f"/vsis3/{bucket}/{prefix}/{bathymetry_dataset}/bathymetry.tif"
+ bathymetry_p = path / Path("bathymetry.nc")
+ bathymetry = bathymetry_from_grid(target_grid, uri=bathymetry_uri, path=bathymetry_p)
+ bed = bed.where(surface > 0.0, bathymetry)
+ bed.name = "bed"
bed.attrs.update({"standard_name": "bedrock_altitude", "units": "m"})
- liafr = surface.rio.clip(glacier_projected.geometry, drop=False)
+ liafr = surface.rio.clip(geometries, drop=False)
liafr = xr.where(liafr.isnull(), 0, 1)
liafr.name = "land_ice_area_fraction_retreat"
liafr.attrs.update({"units": "1"})
liafr = liafr.astype("bool")
- ftt_mask = surface.rio.clip(glacier_projected.geometry, drop=False)
- ftt_mask = xr.where(ftt_mask.isnull(), 1, 0)
+ if forcing_mask == "glacier":
+ print("Forcing mask: 1 outside 0 inside glacier")
+ # rio.clip leaves the surface elevation inside the polygon and NaN
+ # outside. Without the xr.where, the subsequent .astype("bool") would
+ # turn both real elevations AND NaN into True (1 everywhere). Map
+ # NaN -> 1 (outside) and any value -> 0 (inside) explicitly.
+ ftt_mask = surface.rio.clip(geometries, drop=False)
+ ftt_mask = xr.where(ftt_mask.isnull(), 1, 0)
+ elif forcing_mask == "all":
+ print("Forcing mask: 1 everywhere")
+ ftt_mask = xr.ones_like(liafr)
+ else:
+ print("Forcing mask: 0 everywhere")
+ ftt_mask = xr.zeros_like(liafr)
ftt_mask.name = "ftt_mask"
ftt_mask.attrs.update({"units": "1"})
+ if "standard_name" in ftt_mask.attrs:
+ del ftt_mask.attrs["standard_name"]
+
ftt_mask = ftt_mask.astype("bool")
tillwat = xr.zeros_like(bed)
tillwat.name = "tillwat"
tillwat.attrs.update({"units": "m"})
- ds = xr.merge([bed, surface, ice_thickness, liafr, ftt_mask, tillwat])
-
- if velocity_dataset is not ("none" or None):
+ ds = xr.merge([bed, surface, ice_thickness, liafr, ftt_mask, tillwat], compat="no_conflicts")
+ if velocity_dataset not in ("none", None):
v_filename = path / Path(f"obs_{rgi_id}.nc")
- v = glacier_velocities_from_rgi_id(rgi_id, rgi, buffer_distance=5000.0, path=v_filename)
- v = v.rio.reproject_match(surface)
+ v = glacier_velocities_from_grid(target_grid, geometries, path=v_filename)
_v = v["v"].fillna(0)
- ds["tillwat"] = xr.where(_v < 100, 0, xr.where(_v > 500, 2, 1 + (_v - 100) / (500 - 100)))
+ ds["tillwat"] = xr.where(_v < 100, 0, xr.where(_v > 250, 2, 1 + (_v - 100) / (250 - 100)))
ds["tillwat"].attrs.update({"units": "m"})
- ds = xr.merge([ds, _v])
-
- return ds.fillna(0)
+ ds["v"] = v["v"]
+
+ ds = ds.fillna(0)
+
+ # Drop the leftover `band` scalar coord that rxr.open_rasterio leaves on
+ # some inputs (e.g. ice thickness, velocity). If it survives in the merged
+ # Dataset, xarray auto-emits ``coordinates = "band"`` on every variable on
+ # write, which trips up QGIS.
+ ds = ds.drop_vars("band", errors="ignore")
+ for var in ds.data_vars:
+ ds[var].attrs.pop("coordinates", None)
+ ds[var].encoding.pop("coordinates", None)
+
+ # Re-attach the CRS as a CF grid_mapping link on every data variable
+ # and stamp standard_name/units onto x/y so GDAL recognises them as
+ # projected eastings/northings (otherwise the geotransform is dropped).
+ ds = ds.rio.write_crs(dst_crs).rio.write_grid_mapping().rio.write_coordinate_system()
+
+ # Mark the file as CF so GDAL picks the netCDF driver (and reads
+ # grid_mapping → CRS) instead of falling through to the HDF5 driver.
+ ds.attrs["Conventions"] = "CF-1.8"
+
+ for name in ("x", "y", "thickness", "bed", "surface", "tillwat", "ftt_mask", "land_ice_area_fraction_retreat"):
+ if name in ds:
+ ds[name].encoding.update({"_FillValue": None})
+ return ds
diff --git a/pism_terra/glacier/execute.py b/pism_terra/glacier/execute.py
index c3b8cce..104e108 100644
--- a/pism_terra/glacier/execute.py
+++ b/pism_terra/glacier/execute.py
@@ -5,6 +5,7 @@
import warnings
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from pathlib import Path
+from typing import Tuple
from pism_terra.aws import local_to_s3, s3_to_local
@@ -45,17 +46,33 @@ def execute(script: Path):
)
-def ensure_pism_terra_structure(script: Path):
+def ensure_pism_terra_structure(script_uri: str) -> Tuple[str | None, str, Path]:
"""
Ensure that the expected PISM-TERRA structure exists around a run script.
Parameters
----------
- script : Path
- Path to a PISM-TERRA run script.
+ script_uri : str
+ URI or local path to a PISM-TERRA run script.
+
+ Returns
+ -------
+ str | None
+ The S3 bucket inferred from the script_uri to stage files from.
+ str
+ The S3 prefix inferred from the script_uri to stage files from.
+ Path
+ The local path to the PISM-TERRA run script.
"""
- if not script.exists():
- raise ValueError(f"{script} does not exist")
+ script = Path(script_uri)
+
+ staging_bucket = None
+ staging_prefix = "." # No-prefix value that would be computed below
+ if script_uri.startswith("s3://"):
+ # pylint: disable=E1101
+ staging_bucket = str(script.parents[-3].relative_to(script.parents[-2]))
+ staging_prefix = str(script.parents[2].relative_to(script.parents[-3]))
+ script = script.relative_to(script.parents[2])
if (script.parents[0].name != "run_scripts") or not script.parents[1].name.startswith("RGI"):
raise ValueError(
@@ -66,49 +83,49 @@ def ensure_pism_terra_structure(script: Path):
(rgi_dir / "input").mkdir(parents=True, exist_ok=True)
(rgi_dir / "logs").mkdir(parents=True, exist_ok=True)
+ (rgi_dir / "output" / "inverse").mkdir(parents=True, exist_ok=True)
(rgi_dir / "output" / "post_processing").mkdir(parents=True, exist_ok=True)
(rgi_dir / "output" / "scalar").mkdir(parents=True, exist_ok=True)
(rgi_dir / "output" / "spatial").mkdir(parents=True, exist_ok=True)
(rgi_dir / "output" / "state").mkdir(parents=True, exist_ok=True)
+ return staging_bucket, staging_prefix, script
+
def main():
"""CLI Enterypoint to execute a PISM-TERRA run script."""
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.description = "Execute a PISM-TERRA run script."
- parser.add_argument(
+
+ output_bucket = parser.add_argument_group(
+ title="AWS S3 Bucket and prefix to upload the local working directory to at the end of processing."
+ )
+ output_bucket.add_argument(
"--bucket",
- help="AWS S3 Bucket to sync with the local working directory.",
)
- parser.add_argument(
+ output_bucket.add_argument(
"--bucket-prefix",
- help="AWS prefix to sync with the local working directory.",
default="",
)
- parser.add_argument(
- "--job-id",
- help="PISM_TERRA_PREP_ENSEMBLE to stage glacier run files from.",
- )
+
parser.add_argument(
"RUN_SCRIPT",
- help="Path to the PISM run script to execute. If you've provided ``--bucket`` and ``--bucket-prefix``, "
- "this path will need to be relative to ``f's3://{bucket}/{bucket_prefix}/'``.",
- type=Path,
+ help="S3 URL or local path to the PISM run script to execute. If an S3 URI is provided, "
+ "execute assumes a structure like `s3://{some-bucket}/{some-prefix}/RGI*/runs_scripts/*.sh`"
+ "and files under `s3://{some-bucket}/{some-prefix}/` will be downloaded to the local work directory.",
+ type=str,
)
args = parser.parse_args()
work_dir = Path.cwd()
- if args.bucket:
- # FIXME: pism-terra produces hard-coded absolute paths, so things _must_ end up in ${HOME}/data # pylint: disable=W0511
- work_dir /= "data"
- s3_to_local(args.bucket, args.job_id, work_dir)
+ staging_bucket, staging_prefix, local_run_script = ensure_pism_terra_structure(args.RUN_SCRIPT)
+ if staging_bucket and staging_prefix:
- run_script = work_dir / args.RUN_SCRIPT
- ensure_pism_terra_structure(run_script)
+ s3_to_local(staging_bucket, staging_prefix if staging_prefix != "." else "", work_dir)
- execute(run_script)
+ execute(local_run_script)
if args.bucket:
local_to_s3(work_dir, args.bucket, args.bucket_prefix)
diff --git a/pism_terra/glacier/glaciermip4.py b/pism_terra/glacier/glaciermip4.py
new file mode 100644
index 0000000..43703f4
--- /dev/null
+++ b/pism_terra/glacier/glaciermip4.py
@@ -0,0 +1,343 @@
+# Copyright (C) 2026 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+# pylint: disable=too-many-positional-arguments,broad-exception-caught
+
+"""
+GlacierMIP4 preparation.
+
+Region-level analog of :mod:`pism_terra.glacier.s4f`: builds one aggregated
+"complex" per RGI o1 region listed in the setup TOML
+(e.g. ``RGI7_01``, ``RGI7_03``, ``RGI7_07``) whose geometry is the union of
+all native RGI7 complexes in that region, in the per-region CRS the TOML
+specifies. Downstream, ``pism-glacier-stage RGI7_NN ...`` looks the
+aggregate up in ``rgi_c.gpkg`` and stages it like any other complex.
+"""
+
+from __future__ import annotations
+
+import logging
+from argparse import ArgumentParser
+from pathlib import Path
+from typing import Any, Sequence
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+import rioxarray # pylint: disable=unused-import
+import toml
+import xarray as xr
+from pyfiglet import Figlet
+from shapely.geometry import MultiPolygon, Polygon
+
+from pism_terra.download import download_gebco
+from pism_terra.glacier.ice_thickness import (
+ prepare_ice_thickness_frank,
+ prepare_ice_thickness_maffezzoli,
+)
+from pism_terra.glacier.rgi import prepare_rgi
+from pism_terra.log import setup_logging
+
+xr.set_options(keep_attrs=True)
+
+logger = logging.getLogger(__name__)
+
+
+def _add_region_aggregates(
+ complexes: gpd.GeoDataFrame,
+ glaciers: gpd.GeoDataFrame,
+ regions: pd.DataFrame,
+) -> tuple[gpd.GeoDataFrame, gpd.GeoDataFrame]:
+ """
+ Append ``RGI7_NN`` aggregate complexes to *complexes* and tag *glaciers*.
+
+ For every row in ``regions`` (indexed by o1 region code) a single
+ aggregate complex is added whose ``rgi_id`` is ``f"RGI7_{NN}"`` with
+ zero-padded ``NN``, whose geometry is the polygon-union of every
+ native ``-C-NN-`` complex in that region, and whose ``crs`` is taken
+ verbatim from the TOML's ``crs`` field. Each constituent glacier is
+ back-linked to the aggregate via the ``rgi_id_c_aggregate`` column
+ (semicolon-joined to support membership in multiple aggregates).
+
+ Parameters
+ ----------
+ complexes : geopandas.GeoDataFrame
+ Output of :func:`pism_terra.glacier.rgi.prepare_rgi`'s
+ ``rgi_complexes`` layer. Modified in-place via concatenation.
+ glaciers : geopandas.GeoDataFrame
+ The matching ``rgi_glaciers`` layer.
+ regions : pandas.DataFrame
+ TOML's ``[regions]`` table, indexed by o1 code, with columns
+ ``name`` and ``crs``.
+
+ Returns
+ -------
+ tuple of geopandas.GeoDataFrame
+ ``(complexes_with_aggregates, glaciers_with_back_links)``.
+ """
+ if "rgi_id_c_aggregate" not in glaciers.columns:
+ glaciers["rgi_id_c_aggregate"] = ""
+
+ extra_rows: list[dict[str, Any]] = []
+ for r_idx, r_row in regions.iterrows():
+ nn = str(r_idx).zfill(2)
+ agg_name = f"RGI7_{nn}"
+ crs_value = r_row["crs"]
+
+ in_region = glaciers["rgi_id"].str.contains(f"-G-{nn}-", na=False)
+ g_ids = glaciers.loc[in_region, "rgi_id"].tolist()
+ if not g_ids:
+ logger.warning("No RGI v7 glaciers found for region %s", nn)
+ continue
+
+ parent_c_ids = glaciers.loc[glaciers["rgi_id"].isin(g_ids), "rgi_id_c"].dropna().unique()
+ parents = complexes.loc[complexes["rgi_id"].isin(parent_c_ids)]
+ if parents.empty:
+ logger.warning("No parent complexes resolved for %s", agg_name)
+ continue
+
+ merged = parents.geometry.union_all()
+ if isinstance(merged, Polygon):
+ merged = MultiPolygon([merged])
+
+ area_km2 = float(gpd.GeoSeries([merged], crs=complexes.crs).to_crs(crs_value).geometry.area.sum()) / 1e6
+
+ extra_rows.append(
+ {
+ "rgi_id": agg_name,
+ "geometry": merged,
+ "crs": crs_value,
+ "epsg_code": None,
+ "area_km2": area_km2,
+ }
+ )
+
+ # Back-link constituent glaciers. Semicolon-join so a glacier can
+ # belong to multiple aggregates without row duplication.
+ sel = glaciers["rgi_id"].isin(g_ids)
+ existing = glaciers.loc[sel, "rgi_id_c_aggregate"].fillna("")
+ updated = existing.where(existing == "", existing + ";") + agg_name
+ glaciers.loc[sel, "rgi_id_c_aggregate"] = updated
+
+ if extra_rows:
+ extras = gpd.GeoDataFrame(extra_rows, geometry="geometry", crs=complexes.crs)
+ complexes = gpd.GeoDataFrame(
+ pd.concat([complexes, extras], ignore_index=True),
+ geometry="geometry",
+ crs=complexes.crs,
+ )
+ logger.info(
+ "Added %d GlacierMIP4 aggregates: %s",
+ len(extra_rows),
+ ", ".join(r["rgi_id"] for r in extra_rows),
+ )
+
+ return complexes, glaciers
+
+
+def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
+ """
+ Prepare GlacierMIP4 region-level data sets.
+
+ Reads a setup TOML describing one or more RGI o1 regions (with their
+ name and target CRS), builds a per-region ``RGI7_NN`` aggregate
+ complex, and stages the shared inputs (Maffezzoli + Frank ice
+ thickness, GEBCO bathymetry) so that subsequent
+ ``pism-glacier-stage RGI7_NN ...`` invocations succeed.
+
+ Parameters
+ ----------
+ argv : sequence of str or None, optional
+ Command-line arguments (excluding the program name). When
+ ``None``, :data:`sys.argv` is used. Passing ``argv=[]`` is useful
+ from a notebook to bypass ipykernel arguments.
+
+ Returns
+ -------
+ dict
+ Mapping returned by :func:`pism_terra.glacier.rgi.prepare_rgi`
+ with paths to the (now aggregate-augmented) ``rgi_c.gpkg`` and
+ ``rgi_g.gpkg``.
+
+ Notes
+ -----
+ The TOML must contain a ``[regions]`` table keyed by o1 code, e.g.::
+
+ [regions]
+ 1 = {name = "alaska", crs = "EPSG:5936"}
+ 3 = {name = "arctic_canada_north", crs = "EPSG:3413"}
+ 7 = {name = "svalbard_jan_mayen", crs = "EPSG:3413"}
+ """
+ parser = ArgumentParser()
+ parser.add_argument(
+ "--force-overwrite",
+ help="Force re-downloading/re-extracting all source files.",
+ action="store_true",
+ default=False,
+ )
+ parser.add_argument(
+ "--ntasks",
+ help="Number of parallel worker tasks.",
+ type=int,
+ default=8,
+ )
+ parser.add_argument("CONFIG_FILE", nargs=1)
+ parser.add_argument(
+ "OUTPUT_PATH",
+ nargs="?",
+ default=".",
+ help="Root directory for staged outputs.",
+ )
+ args = parser.parse_args(list(argv) if argv is not None else None)
+
+ config_file = args.CONFIG_FILE[0]
+ force_overwrite = args.force_overwrite
+ ntasks = args.ntasks
+ output_path = Path(args.OUTPUT_PATH)
+ output_path.mkdir(parents=True, exist_ok=True)
+
+ setup_logging(output_path / "prepare.log")
+
+ fig = Figlet(font="standard")
+ banner = fig.renderText("pism-terra")
+ logger.info("=" * 120)
+ logger.info("\n%s", banner)
+ logger.info("=" * 120)
+ logger.info("Preparing GlacierMIP4 data")
+ logger.info("-" * 120)
+
+ config = toml.loads(Path(config_file).read_text("utf-8"))
+ regions = pd.DataFrame.from_dict(config["regions"], orient="index")
+ if "crs" not in regions.columns:
+ raise ValueError(
+ 'Every [regions.] entry in the setup TOML must declare a `crs` (e.g. `crs = "EPSG:5936"`).'
+ )
+ regions["region"] = regions.index.astype(str).str.zfill(2) + "_" + regions["name"]
+
+ glacier_path = output_path / "glacier"
+ glacier_path.mkdir(parents=True, exist_ok=True)
+ staging_path = output_path / "staging"
+ staging_path.mkdir(parents=True, exist_ok=True)
+
+ # --- RGI ---
+ rgi_path = glacier_path / "rgi"
+ rgi_path.mkdir(parents=True, exist_ok=True)
+ rgi_staging = staging_path / "rgi"
+ rgi_staging.mkdir(parents=True, exist_ok=True)
+
+ rgi_files = prepare_rgi(
+ regions,
+ glaciers=None,
+ glacier_groups=None,
+ output_path=rgi_path,
+ extract_path=rgi_staging,
+ force_overwrite=force_overwrite,
+ ntasks=ntasks,
+ )
+
+ complexes = gpd.read_file(rgi_files["rgi_complexes"])
+ glaciers = gpd.read_file(rgi_files["rgi_glaciers"])
+
+ # --- Add one ``RGI7_NN`` aggregate complex per region ---
+ complexes, glaciers = _add_region_aggregates(complexes, glaciers, regions)
+ logger.info("Writing aggregated complexes back to %s", rgi_files["rgi_complexes"])
+ complexes.to_file(rgi_files["rgi_complexes"])
+ logger.info("Writing back-linked glaciers to %s", rgi_files["rgi_glaciers"])
+ glaciers.to_file(rgi_files["rgi_glaciers"])
+
+ # --- Ice thickness (Frank + Maffezzoli) ---
+ ice_thickness_path = glacier_path / "ice_thickness"
+ ice_thickness_path.mkdir(parents=True, exist_ok=True)
+ ice_thickness_staging = staging_path / "ice_thickness"
+ ice_thickness_staging.mkdir(parents=True, exist_ok=True)
+
+ frank_path = ice_thickness_path / "frank"
+ frank_path.mkdir(parents=True, exist_ok=True)
+ prepare_ice_thickness_frank(
+ regions.index,
+ complexes=complexes,
+ glaciers=glaciers,
+ output_path=frank_path,
+ extract_path=ice_thickness_staging,
+ rgi_extract_path=rgi_staging,
+ force_overwrite=force_overwrite,
+ ntasks=ntasks,
+ )
+
+ maffezzoli_path = ice_thickness_path / "maffezzoli"
+ maffezzoli_path.mkdir(parents=True, exist_ok=True)
+ prepare_ice_thickness_maffezzoli(
+ regions.index,
+ complexes=complexes,
+ glaciers=glaciers,
+ output_path=maffezzoli_path,
+ extract_path=ice_thickness_staging,
+ force_overwrite=force_overwrite,
+ ntasks=ntasks,
+ )
+
+ # --- GEBCO bathymetry ---
+ gebco_path = glacier_path / "gebco"
+ gebco_path.mkdir(parents=True, exist_ok=True)
+ gebco_staging = staging_path / "gebco"
+ gebco_staging.mkdir(parents=True, exist_ok=True)
+ gebco_nc = download_gebco(target_dir=gebco_staging)
+ cog_gebco_p = gebco_path / "bathymetry.tif"
+
+ ds = xr.open_dataset(gebco_nc, chunks={"lat": 1024, "lon": 1024})
+ da = ds["elevation"].rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=False)
+ if da.rio.crs is None:
+ da = da.rio.write_crs("EPSG:4326")
+ predictor = 3 if np.issubdtype(da.dtype, np.floating) else 2
+ da.rio.to_raster(
+ cog_gebco_p,
+ driver="COG",
+ compress="DEFLATE",
+ predictor=predictor,
+ blocksize=512,
+ bigtiff="YES",
+ overview_resampling="AVERAGE",
+ num_threads="ALL_CPUS",
+ )
+
+ logger.info("GlacierMIP4 preparation complete.")
+ logger.info("Aggregates available: %s", ", ".join(f"RGI7_{str(i).zfill(2)}" for i in regions.index))
+ logger.info("Next: pism-glacier-stage RGI7_NN ")
+ return rgi_files
+
+
+def cli(argv: Sequence[str] | None = None) -> int:
+ """
+ Console entry point.
+
+ Parameters
+ ----------
+ argv : sequence of str or None, optional
+ Command-line arguments (excluding the program name).
+
+ Returns
+ -------
+ int
+ Exit code (``0`` on success).
+ """
+ _ = main(argv=argv)
+ return 0
+
+
+if __name__ == "__main__":
+ __spec__ = None # type: ignore
+ raise SystemExit(cli())
diff --git a/pism_terra/glacier/ice_thickness.py b/pism_terra/glacier/ice_thickness.py
index fc4397d..1e39b88 100644
--- a/pism_terra/glacier/ice_thickness.py
+++ b/pism_terra/glacier/ice_thickness.py
@@ -22,161 +22,545 @@
"""
import logging
+import re
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import as_completed as cf_as_completed
from pathlib import Path
from typing import Literal
-from urllib.parse import urlparse
import geopandas as gpd
import numpy as np
+import pandas as pd
import rasterio
import rioxarray as rxr
import xarray as xr
+from rasterio.mask import mask as rio_mask
from rasterio.merge import merge
-from rasterio.warp import Resampling, calculate_default_transform, reproject
+from rasterio.warp import (
+ Resampling,
+ calculate_default_transform,
+ reproject,
+ transform_bounds,
+)
+from shapely.geometry import box
from tqdm.auto import tqdm
from pism_terra.aws import download_from_s3, s3_to_local
from pism_terra.download import download_archive, extract_archive
-from pism_terra.raster import check_overlap, reproject_file
-from pism_terra.vector import glaciers_in_complex
-from pism_terra.workflow import check_xr_lazy
+from pism_terra.raster import check_overlap
+from pism_terra.workflow import check_xr_lazy, drop_geotransform_attr
logger = logging.getLogger(__name__)
-def get_maffezzoli_url(url_template: str, region: str) -> str:
+def _load_rgi6_links(rgi_extract_path: Path) -> pd.DataFrame:
"""
- Format the given URL template with the provided region.
+ Build a global RGI6 → RGI7-G ID lookup from RGI7 G-archive link CSVs.
+
+ Each RGI7 G regional zip ships a ``*-rgi6_links.csv`` that maps RGI7
+ glacier IDs (``RGI2000-v7.0-G-XX-YYYYY``) to their RGI6 counterparts
+ (``RGI60-XX.YYYYY``). This helper finds every such CSV under
+ *rgi_extract_path*, normalizes the column names, and returns a single
+ concatenated DataFrame with columns ``rgi_id`` (RGI7) and ``rgi6_id``.
Parameters
----------
- url_template : str
- A string with `{region}` and `{outline_type}` placeholders to be replaced.
- region : str
- The region code to insert into the template.
+ rgi_extract_path : pathlib.Path
+ Directory containing extracted RGI7 G archives. Searched recursively
+ for files matching ``*rgi6_links*.csv``.
Returns
-------
- str
- The formatted URL.
+ pandas.DataFrame
+ Two-column frame ``["rgi_id", "rgi6_id"]``. Rows with missing IDs are
+ dropped. Empty if no link files are found.
"""
- return url_template.format(region=region)
-
-
-def prepare_ice_thickness_maffezzoli(
+ csvs = sorted(rgi_extract_path.rglob("*rgi6_links*.csv"))
+ if not csvs:
+ return pd.DataFrame(columns=["rgi_id", "rgi6_id"])
+
+ # Column names vary across RGI7 releases; accept the common spellings.
+ rgi7_aliases = ["rgi_id", "rgi7_id", "rgi70_id", "RGIId", "rgi_id_v7"]
+ rgi6_aliases = ["rgi6_id", "rgi60_id", "RGIId_v6", "rgi_id_v6", "v6_id"]
+
+ frames = []
+ for csv in csvs:
+ df = pd.read_csv(csv)
+ rgi7_col = next((c for c in rgi7_aliases if c in df.columns), None)
+ rgi6_col = next((c for c in rgi6_aliases if c in df.columns), None)
+ if rgi7_col is None or rgi6_col is None:
+ logger.warning(
+ "rgi6_links file %s has unexpected columns %s; skipping",
+ csv,
+ list(df.columns),
+ )
+ continue
+ frames.append(df[[rgi7_col, rgi6_col]].rename(columns={rgi7_col: "rgi_id", rgi6_col: "rgi6_id"}))
+
+ if not frames:
+ return pd.DataFrame(columns=["rgi_id", "rgi6_id"])
+
+ links = pd.concat(frames, ignore_index=True)
+ links = links.dropna(subset=["rgi_id", "rgi6_id"])
+ links["rgi_id"] = links["rgi_id"].astype(str)
+ links["rgi6_id"] = links["rgi6_id"].astype(str)
+ return links
+
+
+def prepare_ice_thickness_frank(
regions: list,
complexes: gpd.GeoDataFrame,
glaciers: gpd.GeoDataFrame,
output_path: Path | str,
extract_path: Path | str,
+ rgi_extract_path: Path | str | None = None,
ntasks: int = 8,
force_overwrite: bool = False,
):
"""
- Download, extract, and merge RGI region ice thickness.
+ Download Frank et al. ice thickness and merge into RGI7 complex rasters.
+
+ The Frank dataset is keyed on RGI6 glacier IDs; this routine pulls the
+ single Figshare archive and matches each per-glacier ``.tif`` to its
+ parent RGI7 complex by **spatial intersection** of the tif footprint
+ with the complex outline. The RGI v6 and v7 numbering schemes are
+ unrelated, and the per-region ``rgi6_links.csv`` files shipped with
+ RGI7 are a sparse subset, so an ID-based join cannot recover the full
+ mapping.
Parameters
----------
regions : list
Region codes (e.g. ``["01", "06"]``).
complexes : geopandas.GeoDataFrame
- Complex outlines with an ``rgi_id`` and ``o1region`` column.
+ Complex outlines with ``rgi_id``, ``crs``, ``geometry``, and (for
+ regular complexes) ``o1region``. Aggregates (no ``o1region``) are
+ merged in a second pass.
glaciers : geopandas.GeoDataFrame
- Glacier outlines with ``rgi_id``, ``rgi_id_c``, and ``o1region`` columns.
- output_path : Path
- Roor directory for output files.
- extract_path : Path or str, optional
- Subdirectory under *output_path* for extracted archives.
- Defaults to ``"ice_thickness"``.
+ Glacier outlines with ``rgi_id``, ``rgi_id_c``, ``o1region``, and
+ (optionally) ``rgi_id_c_aggregate``. Used only to resolve aggregate
+ parent complexes; the per-complex match is purely spatial.
+ output_path : Path or str
+ Root directory for the per-complex / per-aggregate output rasters.
+ extract_path : Path or str
+ Working directory for the Frank archive (``Thk.zip``) and its
+ extracted per-RGI6 ``.tif`` files.
+ rgi_extract_path : Path or str or None, optional
+ Unused; kept for signature parity with :func:`prepare_ice_thickness_maffezzoli`.
ntasks : int, default 8
- Maximum number of parallel workers.
+ Maximum number of parallel workers for the merge phases.
force_overwrite : bool, default False
- If True, re-download the file even if it already exists locally.
+ If True, re-download / re-extract / re-merge.
"""
- url_template: str = "https://zenodo.org/records/17724512/files/RGI70G_rgi{region}.zip?download=1"
+ # Frank et al. (2025) global glacier ice thickness — single archive at
+ # Springer Nature Figshare (file id 57288257), RGI6-keyed per-glacier tifs.
+ url: str = "https://springernature.figshare.com/ndownloader/files/57288257"
extract_path = Path(extract_path)
output_path = Path(output_path)
-
- def _download_and_extract(region):
+ extract_path.mkdir(parents=True, exist_ok=True)
+ _ = rgi_extract_path # unused; kept for signature parity
+
+ archive_dest = extract_path / "Thk.zip"
+ logger.info("Downloading Frank ice thickness from %s", url)
+ archive = download_archive(url, dest=archive_dest, force_overwrite=force_overwrite, verbose=True)
+
+ frank_dir = extract_path / "frank"
+ frank_dir.mkdir(parents=True, exist_ok=True)
+ logger.info("Extracting %s into %s", archive, frank_dir)
+ extract_archive(archive, frank_dir, force_overwrite=force_overwrite, verbose=True)
+
+ # Frank's Thk.zip is a zip-of-zips: a per-region ``RGI-NN_thk.zip`` for
+ # each o1 region. Extract only the regions in scope so we don't unpack
+ # the whole world when, say, only Alaska is requested.
+ region_codes = {str(r).zfill(2) for r in regions}
+ for inner_zip in sorted(frank_dir.glob("RGI-*_thk.zip")):
+ # Filename: RGI-NN_thk.zip — pull NN out of the stem.
+ m = re.match(r"^RGI-(\d{2})_thk$", inner_zip.stem)
+ if not m or m.group(1) not in region_codes:
+ continue
+ out_dir = frank_dir / inner_zip.stem
+ if out_dir.exists() and not force_overwrite and any(out_dir.rglob("RGI60-*.tif")):
+ continue
+ logger.info("Extracting per-region %s into %s", inner_zip.name, out_dir)
+ extract_archive(inner_zip, out_dir, force_overwrite=force_overwrite, verbose=True)
+
+ # Index the extracted Frank tifs by RGI6 id. The RGI7-shipped
+ # ``rgi6_links.csv`` files are a sparse subset (only the renumberings
+ # with non-trivial overlap) and the v6/v7 numbering schemes are
+ # unrelated, so an id-based join can't recover the full mapping.
+ # Instead, build a spatial index over Frank's per-glacier footprints
+ # and find the tifs that physically overlap each RGI7 complex.
+ frank_tifs: dict[str, Path] = {p.stem.removesuffix("_thk"): p for p in frank_dir.rglob("RGI60-*.tif")}
+ if not frank_tifs:
+ logger.error("No RGI60-*.tif files found under %s after extraction", frank_dir)
+ return
+ logger.info("Indexed %d Frank per-glacier tifs", len(frank_tifs))
+
+ def _frank_footprint_4326(item):
"""
- Download and extract ice thickness archive for a single region.
+ Read a Frank tif's bounding box and reproject it to EPSG:4326.
Parameters
----------
- region : str
- Region code (e.g. ``"01"``).
+ item : tuple of (str, pathlib.Path)
+ ``(rgi6_id, path)`` pair from the indexed Frank tif dict.
Returns
-------
- str
- The region code that was processed.
+ dict or None
+ Mapping with ``rgi6_id`` and ``geometry`` (a Polygon in
+ EPSG:4326), or ``None`` if the tif could not be read.
"""
- url = get_maffezzoli_url(url_template, region)
- archive_dest = extract_path / Path(urlparse(url).path).name
- logger.info("Downloading ice thickness for region %s", region)
- archive = download_archive(url, dest=archive_dest, force_overwrite=force_overwrite, verbose=False)
- logger.info("Extracting ice thickness for region %s", region)
- extract_archive(archive, extract_path, force_overwrite=force_overwrite, verbose=False)
- logger.info("Ice thickness download complete for region %s", region)
- return region
-
- MAX_WORKERS = min(ntasks, len(regions))
- failed = []
- with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
- futures = {executor.submit(_download_and_extract, region): region for region in regions}
- pbar = tqdm(cf_as_completed(futures), total=len(futures), desc="Downloading ice thickness")
- for future in pbar:
- region = futures[future]
- try:
- future.result()
- pbar.set_postfix_str(f"region {region} ✓")
- except Exception as e:
- failed.append((region, e))
- pbar.set_postfix_str(f"region {region} ✗")
- for region, err in failed:
- logger.error("Failed ice thickness: region %s with error: %s", region, err)
- logger.info("Starting ice thickness merging for %d regions", len(regions))
+ rgi6_id, tif_path = item
+ try:
+ with rasterio.open(tif_path) as src:
+ if src.crs is None:
+ return None
+ b = src.bounds
+ if src.crs.to_epsg() == 4326:
+ poly = box(b.left, b.bottom, b.right, b.top)
+ else:
+ poly = box(*transform_bounds(src.crs, "EPSG:4326", *b))
+ except Exception: # pylint: disable=broad-exception-caught
+ return None
+ return {"rgi6_id": rgi6_id, "geometry": poly}
+
+ records: list[dict] = []
+ with ThreadPoolExecutor(max_workers=max(1, ntasks)) as executor:
+ for rec in tqdm(
+ executor.map(_frank_footprint_4326, frank_tifs.items()),
+ total=len(frank_tifs),
+ desc="Indexing Frank footprints",
+ ):
+ if rec is not None:
+ records.append(rec)
+ frank_footprints = gpd.GeoDataFrame(records, crs="EPSG:4326")
+ logger.info("Built spatial index over %d Frank footprints", len(frank_footprints))
+
+ # Project complex outlines to 4326 once so the per-complex intersect
+ # call is cheap. The .sindex on frank_footprints accelerates lookups.
+ complexes_4326 = complexes.to_crs("EPSG:4326")
+ _ = frank_footprints.sindex # warm the rtree
+
+ def _reproject_and_merge(input_files, dst_crs, output_file, tmp_label):
+ """
+ Reproject inputs to ``dst_crs`` and write their mosaic to ``output_file``.
+
+ Parameters
+ ----------
+ input_files : list[Path]
+ Source GeoTIFFs to merge.
+ dst_crs : str
+ Target CRS string (e.g. ``"EPSG:32606"``).
+ output_file : Path
+ Target path for the merged GeoTIFF.
+ tmp_label : str
+ Label embedded in temporary reproject filenames so concurrent
+ tasks operating on the same source raster don't collide.
- def _merge_complex(rgi_c_id, region_g, region_code, dst_crs):
+ Returns
+ -------
+ Path
+ ``output_file`` on success.
"""
- Merge glacier thickness rasters for a single complex.
+ reprojected: list[Path] = []
+ for fpath in input_files:
+ with rasterio.open(fpath) as src:
+ if src.crs == rasterio.CRS.from_user_input(dst_crs):
+ reprojected.append(fpath)
+ continue
+ transform, width, height = calculate_default_transform(
+ src.crs, dst_crs, src.width, src.height, *src.bounds
+ )
+ meta = src.meta.copy()
+ meta.update({"crs": dst_crs, "transform": transform, "width": width, "height": height})
+ tmp_path = fpath.parent / f"{fpath.stem}__{tmp_label}_reproj.tif"
+ with rasterio.open(tmp_path, "w", **meta) as dst:
+ reproject(
+ source=rasterio.band(src, 1),
+ destination=rasterio.band(dst, 1),
+ src_transform=src.transform,
+ src_crs=src.crs,
+ dst_transform=transform,
+ dst_crs=dst_crs,
+ resampling=Resampling.bilinear,
+ )
+ reprojected.append(tmp_path)
- Glacier rasters are reprojected to *dst_crs* before merging so that
- glaciers spanning different UTM zones can be combined.
+ # Frank tifs are per-glacier and their bounding boxes overlap at
+ # adjacent-glacier edges. Two quirks dictate the merge call:
+ # 1. The declared nodata (9999) never appears in the actual data
+ # — Frank uses literal 0 for "outside the v6 outline" cells.
+ # Overriding ``nodata=0`` here treats those zero borders as
+ # transparent during merge.
+ # 2. ``method="max"`` then picks the larger value across any
+ # remaining overlap (where two adjacent tifs both have real
+ # thickness near a shared edge).
+ # Without (1), a neighbor's bbox 0s overwrite real thickness on the
+ # downstream side of a glacier, producing the visible "cut" pattern.
+ mosaic, out_transform = merge(reprojected, method="max", nodata=0)
+ with rasterio.open(reprojected[0]) as src:
+ out_meta = src.meta.copy()
+ predictor = 3 if np.issubdtype(mosaic.dtype, np.floating) else 2
+ out_meta.update(
+ {
+ "driver": "COG",
+ "height": mosaic.shape[1],
+ "width": mosaic.shape[2],
+ "transform": out_transform,
+ "compress": "DEFLATE",
+ "predictor": predictor,
+ "level": 6,
+ "blocksize": 512,
+ "overview_resampling": "AVERAGE",
+ "BIGTIFF": "YES",
+ "num_threads": "ALL_CPUS",
+ }
+ )
+ output_file.parent.mkdir(parents=True, exist_ok=True)
+ with rasterio.open(output_file, "w", **out_meta) as dest:
+ dest.write(mosaic)
+
+ for fpath in reprojected:
+ if fpath.stem.endswith("_reproj"):
+ fpath.unlink(missing_ok=True)
+ return output_file
+
+ def _merge_complex(rgi_c_id, o1region, dst_crs):
+ """
+ Mosaic Frank rasters for every RGI6 glacier inside one RGI7 complex.
Parameters
----------
rgi_c_id : str
- The complex outline identifier.
- region_g : geopandas.GeoDataFrame
- Glacier outlines for the region.
- region_code : str
- Region code (e.g. ``"1"``).
+ RGI7 complex identifier (e.g. ``"RGI2000-v7.0-C-01-09429"``).
+ o1region : str or int
+ Region code of the complex; used to build the output subdirectory.
dst_crs : str
- Target CRS for the output (e.g. ``"EPSG:32606"``).
+ Target CRS string for the merged output (e.g. ``"EPSG:32606"``).
Returns
-------
- str or None
- Path to the merged file, or ``None`` if no files found.
+ pathlib.Path or None
+ Path to the merged GeoTIFF, or ``None`` if no Frank rasters
+ resolved to this complex.
"""
- glaciers_list = glaciers_in_complex(rgi_c_id, region_g)
- glaciers_files = [extract_path / Path(f"rgi{region_code}") / Path(f"{g}.tif") for g in glaciers_list]
- glaciers_files = [f for f in glaciers_files if f.exists()]
- if not glaciers_files:
- logger.debug("No thickness files found for complex %s", rgi_c_id)
+ row = complexes_4326.loc[complexes_4326["rgi_id"] == rgi_c_id]
+ if row.empty:
+ logger.warning("Complex %s not found in complexes table; skipping", rgi_c_id)
return None
- logger.debug("Merging %d glacier rasters for complex %s", len(glaciers_files), rgi_c_id)
+ geom = row.geometry.iloc[0]
+ matched = frank_footprints[frank_footprints.intersects(geom)]
+ if matched.empty:
+ logger.warning("Complex %s has no overlapping Frank tifs; skipping", rgi_c_id)
+ return None
+ files = [frank_tifs[r] for r in matched["rgi6_id"].tolist() if r in frank_tifs]
+ merged_path = output_path / Path(f"RGI2000-v7.0-C-{str(o1region).zfill(2)}")
+ merged_file = merged_path / Path(f"{rgi_c_id}_thickness.tif")
+ return _reproject_and_merge(files, dst_crs, merged_file, rgi_c_id)
+
+ def _merge_aggregate(agg_id, dst_crs):
+ """
+ Mosaic the phase-1 per-complex outputs for an aggregate (e.g. ``S4F_AK``).
- # Reproject each glacier raster to the complex's CRS
- reprojected = []
- for fpath in glaciers_files:
+ Parameters
+ ----------
+ agg_id : str
+ Aggregate identifier (e.g. ``"S4F_AK"``).
+ dst_crs : str
+ Target CRS string for the merged output.
+
+ Returns
+ -------
+ pathlib.Path or None
+ Path to the merged aggregate GeoTIFF, or ``None`` if no parent
+ phase-1 thickness files were found.
+ """
+ agg_col = glaciers.get("rgi_id_c_aggregate")
+ if agg_col is None:
+ return None
+ sel = agg_col.fillna("").str.split(";").apply(lambda parts: agg_id in parts)
+ parent_ids = glaciers.loc[sel, "rgi_id_c"].dropna().unique()
+ if len(parent_ids) == 0:
+ logger.debug("No parent complexes resolved for aggregate %s", agg_id)
+ return None
+ parent_files: list[Path] = []
+ for parent_id in parent_ids:
+ for o1_dir in output_path.glob("RGI2000-v7.0-C-*"):
+ p = o1_dir / f"{parent_id}_thickness.tif"
+ if p.exists():
+ parent_files.append(p)
+ break
+ if not parent_files:
+ logger.debug("No phase-1 thickness files for aggregate %s", agg_id)
+ return None
+ merged_file = output_path / Path(agg_id) / Path(f"{agg_id}_thickness.tif")
+ return _reproject_and_merge(parent_files, dst_crs, merged_file, agg_id)
+
+ # Same two-phase scheduling as Maffezzoli: regular complexes first
+ # (they only depend on the Frank tifs), then aggregates (which depend
+ # on the phase-1 per-complex outputs).
+ regular_tasks: list[tuple[str, object, str]] = []
+ aggregate_tasks: list[tuple[str, str]] = []
+ for _, row in complexes.iterrows():
+ crs_value = row.get("crs")
+ if not isinstance(crs_value, str) or not crs_value:
+ logger.warning("Complex %s has no CRS; skipping", row["rgi_id"])
+ continue
+ if pd.notna(row.get("o1region")):
+ regular_tasks.append((row["rgi_id"], row.get("o1region"), crs_value))
+ else:
+ aggregate_tasks.append((row["rgi_id"], crs_value))
+
+ failed: list[tuple[str, Exception]] = []
+ if regular_tasks:
+ with ThreadPoolExecutor(max_workers=min(ntasks, max(1, len(regular_tasks)))) as executor:
+ futures = {
+ executor.submit(_merge_complex, rgi_c_id, o1region, dst_crs): rgi_c_id
+ for rgi_c_id, o1region, dst_crs in regular_tasks
+ }
+ for future in tqdm(cf_as_completed(futures), total=len(futures), desc="Merging Frank thickness"):
+ rgi_c_id = futures[future]
+ try:
+ future.result()
+ except Exception as exc:
+ failed.append((rgi_c_id, exc))
+
+ if aggregate_tasks:
+ with ThreadPoolExecutor(max_workers=min(ntasks, max(1, len(aggregate_tasks)))) as executor:
+ futures = {
+ executor.submit(_merge_aggregate, agg_id, dst_crs): agg_id for agg_id, dst_crs in aggregate_tasks
+ }
+ for future in tqdm(cf_as_completed(futures), total=len(futures), desc="Merging Frank aggregates"):
+ agg_id = futures[future]
+ try:
+ future.result()
+ except Exception as exc:
+ failed.append((agg_id, exc))
+
+ for rgi_c_id, err in failed:
+ logger.error("Failed merging %s: %s", rgi_c_id, err)
+ logger.info("Frank ice thickness merging complete")
+ _ = regions # listed for parity with the Maffezzoli signature
+
+
+def prepare_ice_thickness_maffezzoli(
+ regions: list,
+ complexes: gpd.GeoDataFrame,
+ glaciers: gpd.GeoDataFrame,
+ output_path: Path | str,
+ extract_path: Path | str,
+ ntasks: int = 8,
+ force_overwrite: bool = False,
+):
+ """
+ Download Maffezzoli (IceBoost) ice thickness and clip into RGI7 complex rasters.
+
+ The IceBoost v2.0 "complexes" dataset is computed directly on the RGI7
+ glacier *complex* outlines and shipped as a single archive of per-region,
+ per-UTM-zone mosaics (``complex/rgi{N}/iceboost_..._epsg_{EPSG}.tif``).
+ Because the thickness is already keyed to RGI7 complexes, no per-glacier
+ merging or RGI6/RGI7 reconciliation is needed: each complex raster is just
+ the slice of its region's mosaic(s) that overlaps the complex outline.
+
+ Parameters
+ ----------
+ regions : list
+ Region codes (e.g. ``["01", "06"]``). Used to limit which region
+ mosaics are indexed from the archive.
+ complexes : geopandas.GeoDataFrame
+ Complex outlines with ``rgi_id``, ``crs``, ``geometry``, and (for
+ regular complexes) ``o1region``. Aggregates (no ``o1region``) are
+ merged in a second pass from the per-complex outputs.
+ glaciers : geopandas.GeoDataFrame
+ Glacier outlines with ``rgi_id_c`` and ``rgi_id_c_aggregate`` columns.
+ Used only to resolve aggregate parent complexes in the second pass.
+ output_path : Path or str
+ Root directory for the per-complex / per-aggregate output rasters.
+ extract_path : Path or str
+ Working directory for the IceBoost archive and its extracted mosaics.
+ ntasks : int, default 8
+ Maximum number of parallel workers for the clip / merge phases.
+ force_overwrite : bool, default False
+ If True, re-download / re-extract / re-clip.
+ """
+
+ # Maffezzoli et al. IceBoost v2.0 global glacier ice thickness, computed on
+ # RGI7 complex outlines — single Zenodo archive of per-region/per-UTM-zone
+ # mosaics (record 20463551).
+ url: str = "https://zenodo.org/records/20463551/files/IceBoostv20_complexes_RGI_7.zip?download=1"
+ extract_path = Path(extract_path)
+ output_path = Path(output_path)
+ extract_path.mkdir(parents=True, exist_ok=True)
+
+ archive_dest = extract_path / "IceBoostv20_complexes_RGI_7.zip"
+ logger.info("Downloading Maffezzoli (IceBoost) ice thickness from %s", url)
+ archive = download_archive(url, dest=archive_dest, force_overwrite=force_overwrite, verbose=True)
+
+ maff_dir = extract_path / "maffezzoli"
+ maff_dir.mkdir(parents=True, exist_ok=True)
+ logger.info("Extracting %s into %s", archive, maff_dir)
+ extract_archive(archive, maff_dir, force_overwrite=force_overwrite, verbose=True)
+
+ # Restrict the index to the regions in scope so we don't open mosaics for
+ # the whole world when only a few regions were requested.
+ requested_regions: set[int] = set()
+ for r in regions:
+ try:
+ requested_regions.add(int(str(r)))
+ except (TypeError, ValueError):
+ continue
+
+ # Index the extracted mosaics by RGI primary region. Each entry records the
+ # tif path, its CRS, and its footprint (a box in its own CRS) so the
+ # per-complex overlap test is a cheap geometry intersection.
+ mosaic_re = re.compile(r"_rgi(\d+)_v70G_epsg_(\d+)\.tif$")
+ region_mosaics: dict[int, list[tuple[Path, object, object]]] = {}
+ for tif in sorted(maff_dir.rglob("iceboost_*_rgi*_epsg_*.tif")):
+ m = mosaic_re.search(tif.name)
+ if not m:
+ continue
+ region = int(m.group(1))
+ if requested_regions and region not in requested_regions:
+ continue
+ with rasterio.open(tif) as src:
+ region_mosaics.setdefault(region, []).append((tif, src.crs, box(*src.bounds)))
+ if not region_mosaics:
+ logger.error("No IceBoost mosaics found under %s after extraction", maff_dir)
+ return
+ logger.info("Indexed IceBoost mosaics for regions %s", sorted(region_mosaics))
+
+ # Project complex outlines to 4326 once; each per-mosaic overlap test then
+ # reprojects the single complex geometry into that mosaic's CRS.
+ complexes_4326 = complexes.to_crs("EPSG:4326")
+
+ def _reproject_and_merge(input_files, dst_crs, output_file, tmp_label):
+ """
+ Reproject inputs to ``dst_crs`` and write their mosaic to ``output_file``.
+
+ Parameters
+ ----------
+ input_files : list[Path]
+ Source GeoTIFFs to merge.
+ dst_crs : str
+ Target CRS string (e.g. ``"EPSG:32606"``).
+ output_file : Path
+ Target path for the merged GeoTIFF.
+ tmp_label : str
+ Label embedded in temporary reproject filenames so concurrent
+ tasks operating on the same source raster don't collide.
+
+ Returns
+ -------
+ Path
+ ``output_file`` on success.
+ """
+ reprojected: list[Path] = []
+ for fpath in input_files:
with rasterio.open(fpath) as src:
- if src.crs == rasterio.crs.CRS.from_user_input(dst_crs):
+ if src.crs == rasterio.CRS.from_user_input(dst_crs):
reprojected.append(fpath)
continue
transform, width, height = calculate_default_transform(
@@ -184,7 +568,7 @@ def _merge_complex(rgi_c_id, region_g, region_code, dst_crs):
)
meta = src.meta.copy()
meta.update({"crs": dst_crs, "transform": transform, "width": width, "height": height})
- tmp_path = fpath.parent / f"{fpath.stem}_reproj.tif"
+ tmp_path = fpath.parent / f"{fpath.stem}__{tmp_label}_reproj.tif"
with rasterio.open(tmp_path, "w", **meta) as dst:
reproject(
source=rasterio.band(src, 1),
@@ -200,73 +584,240 @@ def _merge_complex(rgi_c_id, region_g, region_code, dst_crs):
mosaic, out_transform = merge(reprojected)
with rasterio.open(reprojected[0]) as src:
out_meta = src.meta.copy()
+ # Cloud-Optimized GeoTIFF: tiled + DEFLATE + overviews, range-readable
+ # from S3 (so QGIS via /vsis3/ streams without a full download).
+ # predictor=3 for floats, 2 for ints. BIGTIFF=YES for >4 GB outputs.
+ predictor = 3 if np.issubdtype(mosaic.dtype, np.floating) else 2
out_meta.update(
- {"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "transform": out_transform}
+ {
+ "driver": "COG",
+ "height": mosaic.shape[1],
+ "width": mosaic.shape[2],
+ "transform": out_transform,
+ "compress": "DEFLATE",
+ "predictor": predictor,
+ "level": 6,
+ "blocksize": 512,
+ "overview_resampling": "AVERAGE",
+ "BIGTIFF": "YES",
+ "num_threads": "ALL_CPUS",
+ }
)
- merged_path = output_path / Path(f"RGI2000-v7.0-C-{region_code.zfill(2)}")
- merged_path.mkdir(parents=True, exist_ok=True)
-
- merged_file_path = merged_path / Path(f"{rgi_c_id}_thickness.tif")
- with rasterio.open(merged_file_path, "w", **out_meta) as dest:
+ output_file.parent.mkdir(parents=True, exist_ok=True)
+ with rasterio.open(output_file, "w", **out_meta) as dest:
dest.write(mosaic)
- # Clean up temporary reprojected files
for fpath in reprojected:
if fpath.stem.endswith("_reproj"):
fpath.unlink(missing_ok=True)
+ return output_file
+
+ def _clip_to_geom(mosaic_path, geom, tmp_path):
+ """
+ Crop the thickness band of a mosaic to ``geom`` and write the window.
- return str(merged_path)
-
- # Build list of all (rgi_c_id, region_g, region_code) tasks across regions
- merge_tasks = []
- for region in regions:
- region_c = complexes[complexes["o1region"] == region.zfill(2)]
- region_g = glaciers[glaciers["o1region"] == region.zfill(2)]
- for _, row in region_c.iterrows():
- merge_tasks.append((row["rgi_id"], region_g, region, row["epsg"]))
-
- with ThreadPoolExecutor(max_workers=min(ntasks, max(1, len(merge_tasks)))) as executor:
- futures = {
- executor.submit(
- _merge_complex, rgi_c_id=rgi_c_id, region_g=region_g, region_code=region_code, dst_crs=dst_crs
- ): rgi_c_id
- for rgi_c_id, region_g, region_code, dst_crs in merge_tasks
- }
- failed = []
- for future in tqdm(cf_as_completed(futures), total=len(futures), desc="Merging ice thickness"):
- rgi_c_id = futures[future]
+ IceBoost mosaics are multi-band (``thickness``, ``thickness_err``,
+ ``jensen_gap``, ``h_wgs84``, ``n_geoid``); only band 1 (``thickness``)
+ is kept so the per-complex output stays single-band like the rest of
+ the pipeline expects.
+
+ Parameters
+ ----------
+ mosaic_path : pathlib.Path
+ Source regional mosaic GeoTIFF.
+ geom : shapely.geometry.base.BaseGeometry
+ Complex outline expressed in *mosaic_path*'s CRS.
+ tmp_path : pathlib.Path
+ Destination for the cropped GeoTIFF.
+
+ Returns
+ -------
+ pathlib.Path or None
+ ``tmp_path`` on success, or ``None`` if ``geom`` does not overlap
+ the raster (``rasterio.mask`` raises in that case).
+ """
+ with rasterio.open(mosaic_path) as src:
try:
- future.result()
- except Exception as exc:
- failed.append((rgi_c_id, exc))
- for rgi_c_id, err in failed:
- logger.error("Failed merging %s: %s", rgi_c_id, err)
+ out_image, out_transform = rio_mask(src, [geom], crop=True, all_touched=True, indexes=[1])
+ except ValueError:
+ return None # geometry doesn't overlap this mosaic
+ out_meta = src.meta.copy()
+ out_meta.update(
+ {"count": 1, "height": out_image.shape[1], "width": out_image.shape[2], "transform": out_transform}
+ )
+ tmp_path.parent.mkdir(parents=True, exist_ok=True)
+ with rasterio.open(tmp_path, "w", **out_meta) as dst:
+ dst.write(out_image)
+ return tmp_path
+
+ def _merge_complex(rgi_c_id, o1region, dst_crs):
+ """
+ Build a per-complex thickness raster by clipping the region mosaic(s).
+
+ Phase-1: find the region's IceBoost mosaic(s) that overlap the complex
+ outline, crop each to the complex extent, and reproject/mosaic the
+ crop(s) into one per-complex GeoTIFF in ``dst_crs``.
+
+ Parameters
+ ----------
+ rgi_c_id : str
+ RGI7 complex identifier (e.g. ``"RGI2000-v7.0-C-01-09429"``).
+ o1region : str or int
+ Region code of the complex; used to build the output subdirectory.
+ dst_crs : str
+ Target CRS string for the merged output (e.g. ``"EPSG:32606"``).
+
+ Returns
+ -------
+ pathlib.Path or None
+ Path to the merged GeoTIFF, or ``None`` if no mosaic overlapped
+ this complex.
+ """
+ candidates = region_mosaics.get(int(o1region), [])
+ if not candidates:
+ logger.warning("No IceBoost mosaics for region %s (complex %s); skipping", o1region, rgi_c_id)
+ return None
+ row = complexes_4326.loc[complexes_4326["rgi_id"] == rgi_c_id]
+ if row.empty:
+ logger.warning("Complex %s not found in complexes table; skipping", rgi_c_id)
+ return None
+ geom_4326 = row.geometry.iloc[0]
+
+ clipped_files: list[Path] = []
+ for idx, (mosaic_path, mcrs, mbounds) in enumerate(candidates):
+ geom_m = gpd.GeoSeries([geom_4326], crs="EPSG:4326").to_crs(mcrs).iloc[0]
+ if not geom_m.intersects(mbounds):
+ continue
+ tmp_path = mosaic_path.parent / f"{rgi_c_id}__clip{idx}.tif"
+ clip = _clip_to_geom(mosaic_path, geom_m, tmp_path)
+ if clip is not None:
+ clipped_files.append(clip)
+ if not clipped_files:
+ logger.warning("Complex %s does not overlap any IceBoost mosaic; skipping", rgi_c_id)
+ return None
+
+ merged_path = output_path / Path(f"RGI2000-v7.0-C-{str(o1region).zfill(2)}")
+ merged_file = merged_path / Path(f"{rgi_c_id}_thickness.tif")
+ try:
+ return _reproject_and_merge(clipped_files, dst_crs, merged_file, rgi_c_id)
+ finally:
+ for fpath in clipped_files:
+ fpath.unlink(missing_ok=True)
+
+ def _merge_aggregate(agg_id, dst_crs):
+ """
+ Build an aggregate thickness raster from per-complex outputs.
+
+ Phase-2: looks up the parent complexes that own glaciers tagged with
+ ``agg_id`` (via the ``rgi_id_c_aggregate`` column), reads each parent's
+ already-produced ``_thickness.tif`` from phase 1, and
+ reprojects/mosaics them into one aggregate GeoTIFF.
+
+ Parameters
+ ----------
+ agg_id : str
+ Aggregate identifier (e.g. ``"S4F_AK"``).
+ dst_crs : str
+ Target CRS string for the merged output.
+
+ Returns
+ -------
+ pathlib.Path or None
+ Path to the merged GeoTIFF, or ``None`` if no parent thickness
+ files (from phase 1) were found on disk.
+ """
+ agg_col = glaciers.get("rgi_id_c_aggregate")
+ if agg_col is None:
+ return None
+ sel = agg_col.fillna("").str.split(";").apply(lambda parts: agg_id in parts)
+ parent_ids = glaciers.loc[sel, "rgi_id_c"].dropna().unique()
+ if len(parent_ids) == 0:
+ logger.debug("No parent complexes resolved for aggregate %s", agg_id)
+ return None
+
+ parent_files: list[Path] = []
+ for parent_id in parent_ids:
+ # Per-region subdirs are output_path/RGI2000-v7.0-C-XX/_thickness.tif
+ for o1_dir in output_path.glob("RGI2000-v7.0-C-*"):
+ p = o1_dir / f"{parent_id}_thickness.tif"
+ if p.exists():
+ parent_files.append(p)
+ break
+ if not parent_files:
+ logger.debug("No parent thickness files found for aggregate %s", agg_id)
+ return None
+
+ merged_file = output_path / Path(agg_id) / Path(f"{agg_id}_thickness.tif")
+ return _reproject_and_merge(parent_files, dst_crs, merged_file, agg_id)
+
+ # Split tasks: regular complexes (have an o1region) run first; aggregates
+ # depend on their parents' merged outputs and run after.
+ regular_tasks: list[tuple[str, object, str]] = []
+ aggregate_tasks: list[tuple[str, str]] = []
+ for _, row in complexes.iterrows():
+ crs_value = row.get("crs")
+ if not isinstance(crs_value, str) or not crs_value:
+ logger.warning("Complex %s has no CRS; skipping", row["rgi_id"])
+ continue
+ if pd.notna(row.get("o1region")):
+ regular_tasks.append((row["rgi_id"], row.get("o1region"), crs_value))
+ else:
+ aggregate_tasks.append((row["rgi_id"], crs_value))
+
+ failed: list[tuple[str, Exception]] = []
+ if regular_tasks:
+ with ThreadPoolExecutor(max_workers=min(ntasks, max(1, len(regular_tasks)))) as executor:
+ futures = {
+ executor.submit(_merge_complex, rgi_c_id, o1region, dst_crs): rgi_c_id
+ for rgi_c_id, o1region, dst_crs in regular_tasks
+ }
+ for future in tqdm(cf_as_completed(futures), total=len(futures), desc="Clipping ice thickness"):
+ rgi_c_id = futures[future]
+ try:
+ future.result()
+ except Exception as exc:
+ failed.append((rgi_c_id, exc))
+
+ if aggregate_tasks:
+ with ThreadPoolExecutor(max_workers=min(ntasks, max(1, len(aggregate_tasks)))) as executor:
+ futures = {
+ executor.submit(_merge_aggregate, agg_id, dst_crs): agg_id for agg_id, dst_crs in aggregate_tasks
+ }
+ for future in tqdm(cf_as_completed(futures), total=len(futures), desc="Merging aggregates"):
+ agg_id = futures[future]
+ try:
+ future.result()
+ except Exception as exc:
+ failed.append((agg_id, exc))
+
+ for rgi_c_id, err in failed:
+ logger.error("Failed merging %s: %s", rgi_c_id, err)
logger.info("Ice thickness merging complete")
def get_ice_thickness(
- glacier,
+ rgi_id: str,
target_grid: xr.Dataset | xr.DataArray,
- dataset: Literal["millan", "maffezzoli"] = "maffezzoli",
+ dataset: Literal["frank", "maffezzoli", "millan"] = "maffezzoli",
path: str | Path = "input_files",
**kwargs,
-):
+) -> xr.DataArray:
"""
Prepare ice thickness data for a given glacier and target grid.
This function dispatches to a dataset-specific loader to prepare an
ice thickness field interpolated to the resolution and bounds of a
- specified target grid. Currently only the "millan" dataset is supported.
+ specified target grid.
Parameters
----------
- glacier : geopandas.GeoDataFrame or geopandas.GeoSeries
- Glacier geometry to match against ice thickness tiles.
+ rgi_id : str
+ Glacier identifier (e.g., ``"RGI2000-v7.0-C-06-00014"``) used to locate
+ the relevant ice thickness tiles.
target_grid : xarray.Dataset or xarray.DataArray
Grid to which the output ice thickness will be interpolated.
- dataset : str, optional
- The name of the ice thickness dataset to use. Currently only "millan" is implemented.
- Default is "millan".
+ dataset : {"millan", "maffezzoli"}, default ``"maffezzoli"``
+ The name of the ice thickness dataset to use.
path : str or pathlib.Path, default ``"input_files"``
Working directory used by helper routines to cache/write intermediate rasters/grids.
**kwargs
@@ -284,10 +835,12 @@ def get_ice_thickness(
If the specified dataset is not supported.
"""
logger.info("Getting ice thickness from dataset '%s'", dataset)
- if dataset == "maffezzoli":
- thickness = get_ice_thickness_maffezzoli(glacier, target_grid, path=path, **kwargs)
+ if dataset == "frank":
+ thickness = get_ice_thickness_frank(rgi_id, target_grid, path=path, **kwargs)
+ elif dataset == "maffezzoli":
+ thickness = get_ice_thickness_maffezzoli(rgi_id, target_grid, path=path, **kwargs)
elif dataset == "millan":
- thickness = get_ice_thickness_millan(glacier, target_grid, path=path, **kwargs)
+ thickness = get_ice_thickness_millan(rgi_id, target_grid, path=path, **kwargs)
else:
raise NotImplementedError(f"Ice thickness dataset '{dataset}' not implemented.")
thickness = thickness.where(thickness > 0, 0)
@@ -296,16 +849,17 @@ def get_ice_thickness(
return thickness
-def get_ice_thickness_maffezzoli(
- glacier, target_grid: xr.Dataset | xr.DataArray, path: str | Path = "input_files", **kwargs
+def get_ice_thickness_frank(
+ rgi_id: str, target_grid: xr.Dataset | xr.DataArray, path: str | Path = "input_files", **kwargs
):
"""
- Load and interpolate Maffezzoli et al. (2026) ice thickness data to a target grid.
+ Load and interpolate Frank et al. (2026) ice thickness data to a target grid.
Parameters
----------
- glacier : geopandas.GeoDataFrame or geopandas.GeoSeries
- Geometry of the glacier to extract overlapping thickness rasters.
+ rgi_id : str
+ Glacier identifier (e.g., ``"RGI2000-v7.0-C-06-00014"``) used to locate
+ the overlapping Maffezzoli thickness rasters.
target_grid : xarray.Dataset or xarray.DataArray
Target grid to which ice thickness should be interpolated.
path : str or pathlib.Path, default ``"input_files"``
@@ -318,63 +872,74 @@ def get_ice_thickness_maffezzoli(
Returns
-------
xarray.DataArray
- Interpolated and summed ice thickness field on the target grid.
+ Interpolated ice thickness field on the target grid.
Notes
-----
- Uses `rioxarray` to load and project raster files.
- - All overlapping rasters are summed to produce the final thickness field.
- - Assumes a fixed reprojected resolution of 50 meters.
+ - Assumes a fixed reprojected resolution of 100 meters.
"""
force_overwrite: bool = bool(kwargs.pop("force_overwrite", False))
bucket: str = kwargs.pop("bucket", "pism-cloud-data")
+ prefix: str = kwargs.pop("prefix", "glacier")
out_dir = Path(path)
- thickness_file = out_dir / "thickness_maffezzoli.nc"
+ thickness_file = out_dir / f"thickness_frank_{rgi_id}.nc"
if (not check_xr_lazy(thickness_file)) or force_overwrite:
thickness_file.unlink(missing_ok=True)
- rgi_id = glacier.iloc[0]["rgi_id"]
- o1region = glacier.iloc[0]["o1region"]
- region = f"RGI2000-v7.0-C-{o1region}"
- s3_uri = f"s3://{bucket}/glaciers/ice_thickness/maffezzoli/{region}/{rgi_id}_thickness.tif"
+ # Regular complex IDs strip the trailing glacier-number segment to get
+ # the per-region subdir; aggregate IDs (no hyphens) live under their
+ # own name.
+ region = "-".join(rgi_id.split("-")[:-1]) or rgi_id
+ s3_uri = f"s3://{bucket}/{prefix}/ice_thickness/frank/{region}/{rgi_id}_thickness.tif"
local_tif = out_dir / f"{rgi_id}_thickness.tif"
- logger.info("Downloading Maffezzoli thickness for %s from S3", rgi_id)
+ print(f"Downloading Frank thickness from {s3_uri}", flush=True)
download_from_s3(s3_uri, local_tif)
- logger.info("Reprojecting thickness to %s", kwargs["target_crs"])
- projected_file = reproject_file(local_tif, dst_crs=kwargs["target_crs"], resolution=50)
- da = rxr.open_rasterio(projected_file).sel(band=1).drop_vars("band")
- logger.info("Interpolating thickness to target grid")
- thickness = da.interp_like(target_grid)
+ logger.info("Reprojecting and aligning thickness to target grid")
+ da = rxr.open_rasterio(local_tif).sel(band=1).drop_vars("band")
+ thickness = da.rio.reproject_match(target_grid, resampling=Resampling.bilinear)
thickness.rio.write_nodata(None, inplace=True)
thickness.name = "thickness"
+ thickness = thickness.rio.write_crs(target_grid.rio.crs).rio.write_grid_mapping()
+ # Drop the GeoTransform so GDAL/QGIS derive the (top-down) transform from
+ # the ascending y coordinate instead of rendering the raster upside-down.
+ drop_geotransform_attr(thickness)
thickness.to_netcdf(thickness_file)
logger.info("Thickness saved to %s", thickness_file)
- else:
- logger.info("Using cached thickness file %s", thickness_file)
-
- thickness = xr.open_dataarray(thickness_file)
-
+ # Return the in-memory result; CRS doesn't always survive the netCDF
+ # round-trip, and we just computed a CRS-correct value.
+ return thickness
+
+ logger.info("Using cached thickness file %s", thickness_file)
+ with xr.open_dataset(thickness_file) as ds:
+ src_crs = ds.rio.crs
+ if src_crs is None and "spatial_ref" in ds.coords:
+ sr = ds["spatial_ref"]
+ src_crs = sr.attrs.get("crs_wkt") or sr.attrs.get("spatial_ref")
+ thickness = ds["thickness"].load()
+ if src_crs is None:
+ # Last resort: trust target_grid's CRS (the cache was generated against
+ # an aligned grid, so this is correct by construction).
+ src_crs = target_grid.rio.crs
+ thickness = thickness.rio.write_crs(src_crs)
return thickness
-def get_ice_thickness_millan(
- glacier, target_grid: xr.Dataset | xr.DataArray, path: str | Path = "input_files", **kwargs
+def get_ice_thickness_maffezzoli(
+ rgi_id: str, target_grid: xr.Dataset | xr.DataArray, path: str | Path = "input_files", **kwargs
):
"""
- Load and interpolate Millan et al. (2022) ice thickness data to a target grid.
-
- This function identifies all Millan ice thickness raster files that overlap
- the input glacier geometry, reprojects them to the specified CRS and resolution,
- interpolates them onto the target grid, and returns the summed result.
+ Load and interpolate Maffezzoli et al. (2026) ice thickness data to a target grid.
Parameters
----------
- glacier : geopandas.GeoDataFrame or geopandas.GeoSeries
- Geometry of the glacier to extract overlapping thickness rasters.
+ rgi_id : str
+ Glacier identifier (e.g., ``"RGI2000-v7.0-C-06-00014"``) used to locate
+ the overlapping Maffezzoli thickness rasters.
target_grid : xarray.Dataset or xarray.DataArray
Target grid to which ice thickness should be interpolated.
path : str or pathlib.Path, default ``"input_files"``
@@ -387,116 +952,188 @@ def get_ice_thickness_millan(
Returns
-------
xarray.DataArray
- Interpolated and summed ice thickness field on the target grid.
+ Interpolated ice thickness field on the target grid.
Notes
-----
- Uses `rioxarray` to load and project raster files.
- - All overlapping rasters are summed to produce the final thickness field.
- - Assumes a fixed reprojected resolution of 50 meters.
+ - Assumes a fixed reprojected resolution of 100 meters.
"""
force_overwrite: bool = bool(kwargs.pop("force_overwrite", False))
bucket: str = kwargs.pop("bucket", "pism-cloud-data")
+ prefix: str = kwargs.pop("prefix", "glacier")
out_dir = Path(path)
- thickness_file = out_dir / "thickness_millan.nc"
+ thickness_file = out_dir / f"thickness_maffezzoli_{rgi_id}.nc"
if (not check_xr_lazy(thickness_file)) or force_overwrite:
-
thickness_file.unlink(missing_ok=True)
- logger.info("Downloading Millan thickness data from S3")
- # Could tweak this to only pull the relevant regions instead of all of it
- s3_to_local(bucket, prefix="millan", dest="data/ice_thickness/millan")
- ice_thickness_files = list(Path("data/ice_thickness/millan").rglob("THICKNESS_*.tif"))
- logger.info("Found %d Millan thickness tiles, checking overlap", len(ice_thickness_files))
- with ThreadPoolExecutor() as executor:
- futures = [executor.submit(check_overlap, path, glacier) for path in ice_thickness_files]
+ # Regular complex IDs strip the trailing glacier-number segment to get
+ # the per-region subdir; aggregate IDs (no hyphens) live under their
+ # own name.
+ region = "-".join(rgi_id.split("-")[:-1]) or rgi_id
+ s3_uri = f"s3://{bucket}/{prefix}/ice_thickness/maffezzoli/{region}/{rgi_id}_thickness.tif"
+ local_tif = out_dir / f"{rgi_id}_thickness.tif"
+ print(f"Downloading Maffezzoli thickness from {s3_uri}", flush=True)
+ download_from_s3(s3_uri, local_tif)
- overlapping_rasters = [f.result() for f in cf_as_completed(futures) if f.result() is not None]
+ logger.info("Reprojecting and aligning thickness to target grid")
+ da = rxr.open_rasterio(local_tif).sel(band=1).drop_vars("band")
+ thickness = da.rio.reproject_match(target_grid, resampling=Resampling.bilinear)
+ thickness.rio.write_nodata(None, inplace=True)
+ thickness.name = "thickness"
+ thickness = thickness.rio.write_crs(target_grid.rio.crs).rio.write_grid_mapping()
+ # Drop the GeoTransform so GDAL/QGIS derive the (top-down) transform from
+ # the ascending y coordinate instead of rendering the raster upside-down.
+ drop_geotransform_attr(thickness)
+ thickness.to_netcdf(thickness_file)
+ logger.info("Thickness saved to %s", thickness_file)
+ # Return the in-memory result; CRS doesn't always survive the netCDF
+ # round-trip, and we just computed a CRS-correct value.
+ return thickness
+
+ logger.info("Using cached thickness file %s", thickness_file)
+ with xr.open_dataset(thickness_file) as ds:
+ src_crs = ds.rio.crs
+ if src_crs is None and "spatial_ref" in ds.coords:
+ sr = ds["spatial_ref"]
+ src_crs = sr.attrs.get("crs_wkt") or sr.attrs.get("spatial_ref")
+ thickness = ds["thickness"].load()
+ if src_crs is None:
+ # Last resort: trust target_grid's CRS (the cache was generated against
+ # an aligned grid, so this is correct by construction).
+ src_crs = target_grid.rio.crs
+ thickness = thickness.rio.write_crs(src_crs)
+ return thickness
- logger.info("Found %d overlapping rasters, reprojecting and interpolating", len(overlapping_rasters))
- thicknesses = []
- for k, p in enumerate(overlapping_rasters):
- if p is not None:
- projected_file = reproject_file(p, dst_crs=kwargs["target_crs"], resolution=50)
- da = rxr.open_rasterio(projected_file).squeeze().drop_vars("band", errors="ignore")
- thickness = da.interp_like(target_grid)
- thickness.rio.write_nodata(None, inplace=True)
- thickness.name = "thickness"
- thickness["raster"] = k
- thicknesses.append(thickness)
- thickness = xr.concat(thicknesses, dim="raster").sum(dim="raster")
- thickness.to_netcdf(thickness_file)
- logger.info("Millan thickness saved to %s", thickness_file)
- else:
- logger.info("Using cached thickness file %s", thickness_file)
+def _millan_region_dir(rgi_id: str) -> str:
+ """
+ Map an RGI ID to its Millan region directory.
- thickness = xr.open_dataarray(thickness_file)
+ Millan tiles are organised by RGI primary region under ``RGI-{N}/``,
+ with the Central Asia regions (13–15) merged into ``RGI-13-15/``.
- return thickness
+ Parameters
+ ----------
+ rgi_id : str
+ Glacier identifier, e.g. ``"RGI2000-v7.0-C-01-04374"``.
+ Returns
+ -------
+ str
+ Millan region subdirectory name (e.g. ``"RGI-1"``, ``"RGI-13-15"``).
+ """
+ parts = rgi_id.split("-")
+ if len(parts) < 4:
+ raise ValueError(f"Cannot derive Millan region from RGI ID {rgi_id!r}")
+ region = int(parts[3])
+ if region in (13, 14, 15):
+ return "RGI-13-15"
+ return f"RGI-{region}"
-def get_ice_thickness_farinotti(glacier):
+
+def get_ice_thickness_millan(
+ rgi_id: str, target_grid: xr.Dataset | xr.DataArray, path: str | Path = "input_files", **kwargs
+):
"""
- Load and interpolate Farniotti et al (2019) ice thickness data to a target grid.
+ Load and interpolate Millan et al. (2022) ice thickness data to a target grid.
- This function identifies all Millan ice thickness raster files that overlap
- the input glacier geometry, reprojects them to the specified CRS and resolution,
- interpolates them onto the target grid, and returns the summed result.
+ Downloads only the Millan tiles for the RGI primary region matching
+ ``rgi_id``, filters them by overlap against the supplied glacier
+ geometry, reprojects each overlapping tile onto ``target_grid`` and
+ returns the per-pixel sum.
Parameters
----------
- glacier : geopandas.GeoDataFrame or geopandas.GeoSeries
- Geometry of the glacier to extract overlapping thickness rasters.
+ rgi_id : str
+ Glacier identifier (e.g., ``"RGI2000-v7.0-C-01-04374"``) used to
+ locate the per-region tile directory.
+ target_grid : xarray.Dataset or xarray.DataArray
+ Target grid to which ice thickness should be interpolated.
+ path : str or pathlib.Path, default ``"input_files"``
+ Working directory used by helper routines to cache/write intermediate rasters/grids.
+ **kwargs
+ Additional keyword arguments. Recognised keys:
+
+ - geometries : geopandas.GeoSeries / GeoDataFrame / iterable
+ Glacier outline used to filter overlapping tiles. Required.
+ - bucket : str, default ``"pism-cloud-data"``
+ - prefix : str, default ``"rgi/glacier"``
+ S3 prefix; tiles are read from
+ ``{prefix}/ice_thickness/millan/RGI-{region}/``.
+ - force_overwrite : bool, default False
Returns
-------
xarray.DataArray
Interpolated and summed ice thickness field on the target grid.
-
- Notes
- -----
- - Uses `rioxarray` to load and project raster files.
- - All overlapping rasters are summed to produce the final thickness field.
- - Assumes a fixed reprojected resolution of 50 meters.
"""
- path = Path("data/ice_thickness")
- path.mkdir(parents=True, exist_ok=True)
-
- region = glacier["o1region"]
- url = f"https://www.research-collection.ethz.ch/bitstream/handle/20.500.11850/315707/composite_thickness_RGI60-{region}.zip"
- archive = download_archive(url)
- extract_archive(archive, extract_to=path)
-
- ice_thickness_files = list(Path(f"data/ice_thickness/RGI60-{region}").rglob("*.tif"))
+ force_overwrite: bool = bool(kwargs.pop("force_overwrite", False))
+ bucket: str = kwargs.pop("bucket", "pism-cloud-data")
+ prefix: str = kwargs.pop("prefix", "rgi/glacier")
+ geometries = kwargs.pop("geometries", None)
+ if geometries is None:
+ raise ValueError("get_ice_thickness_millan requires `geometries` for tile overlap check")
+ # raster_overlaps_glacier's GeoSeries branch has a missing to_crs (FIXME);
+ # wrapping as a GeoDataFrame routes through the branch that handles CRS
+ # correctly, since Millan tiles and the glacier are in different CRSes.
+ if isinstance(geometries, gpd.GeoSeries):
+ geometries = gpd.GeoDataFrame(geometry=geometries, crs=geometries.crs)
- with ThreadPoolExecutor() as executor:
- futures = [executor.submit(check_overlap, path, glacier) for path in ice_thickness_files]
+ out_dir = Path(path)
+ thickness_file = out_dir / f"thickness_millan_{rgi_id}.nc"
- overlapping_rasters = [f.result() for f in cf_as_completed(futures) if f.result() is not None]
+ if (not check_xr_lazy(thickness_file)) or force_overwrite:
- # Step 1: List all .tif files
- tif_files = overlapping_rasters
+ thickness_file.unlink(missing_ok=True)
+ region_dir = _millan_region_dir(rgi_id)
+ s3_prefix = f"{prefix}/ice_thickness/millan/{region_dir}"
+ local_root = Path("data/ice_thickness/millan") / region_dir
+ logger.info("Downloading Millan thickness tiles from s3://%s/%s", bucket, s3_prefix)
+ s3_to_local(bucket, prefix=s3_prefix, dest=local_root)
+ ice_thickness_files = list(local_root.rglob("THICKNESS_*.tif"))
+ logger.info("Found %d Millan thickness tiles in %s, checking overlap", len(ice_thickness_files), region_dir)
- # Step 2: Open all files as datasets
- src_files_to_mosaic = [rasterio.open(fp) for fp in tif_files]
+ with ThreadPoolExecutor() as executor:
+ futures = [executor.submit(check_overlap, tile, geometries) for tile in ice_thickness_files]
+ overlapping_rasters = [f.result() for f in cf_as_completed(futures) if f.result() is not None]
- # Step 3: Merge them
- mosaic, out_transform = merge(src_files_to_mosaic)
+ logger.info("Found %d overlapping rasters, reprojecting and interpolating", len(overlapping_rasters))
+ thicknesses = []
+ for k, p in enumerate(overlapping_rasters):
+ if p is not None:
+ da = rxr.open_rasterio(p).squeeze().drop_vars("band", errors="ignore")
+ thickness = da.rio.reproject_match(target_grid, resampling=Resampling.bilinear)
+ thickness.rio.write_nodata(None, inplace=True)
+ thickness.name = "thickness"
+ thickness["raster"] = k
+ thicknesses.append(thickness)
- # Step 4: Get metadata from first file, update with new shape and transform
- out_meta = src_files_to_mosaic[0].meta.copy()
- out_meta.update(
- {"driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "transform": out_transform}
- )
+ thickness = xr.concat(thicknesses, dim="raster").sum(dim="raster")
+ thickness = thickness.rio.write_crs(target_grid.rio.crs).rio.write_grid_mapping()
+ # Drop the GeoTransform so GDAL/QGIS derive the (top-down) transform from
+ # the ascending y coordinate instead of rendering the raster upside-down.
+ drop_geotransform_attr(thickness)
+ thickness.to_netcdf(thickness_file)
+ logger.info("Millan thickness saved to %s", thickness_file)
+ return thickness
+
+ logger.info("Using cached thickness file %s", thickness_file)
+ with xr.open_dataset(thickness_file) as ds:
+ src_crs = ds.rio.crs
+ if src_crs is None and "spatial_ref" in ds.coords:
+ sr = ds["spatial_ref"]
+ src_crs = sr.attrs.get("crs_wkt") or sr.attrs.get("spatial_ref")
+ thickness = ds["thickness"].load()
+ if src_crs is None:
+ src_crs = target_grid.rio.crs
+ thickness = thickness.rio.write_crs(src_crs)
- # Step 5: Write the result to disk
- with rasterio.open("merged.tif", "w", **out_meta) as dest:
- dest.write(mosaic)
+ return thickness
def add_malaspina_bed(
@@ -553,7 +1190,7 @@ def add_malaspina_bed(
ds["thickness"] = ds["thickness"].where(ds["thickness"] > 0.0, 0.0)
ds["surface"] = ds["surface"].where(ds["thickness"] > 0.0, 0.0)
- ds["surface"].attrs.update({"standard_name": "land_ice_elevation", "units": "m"})
+ ds["surface"].attrs.update({"standard_name": "surface_altitude", "units": "m"})
ds["thickness"].attrs.update({"standard_name": "land_ice_thickness", "units": "m"})
diff --git a/pism_terra/glacier/observations.py b/pism_terra/glacier/observations.py
index ab47687..66a5959 100644
--- a/pism_terra/glacier/observations.py
+++ b/pism_terra/glacier/observations.py
@@ -23,21 +23,121 @@
Prepare observations.
"""
+import collections
+from functools import lru_cache
from pathlib import Path
import geopandas as gpd
+import numpy as np
import rasterio
import rioxarray as rxr
import xarray as xr
from pyproj import Transformer
+from rasterio.enums import Resampling
+from scipy.interpolate import RegularGridInterpolator
+from shapely.geometry import box as _shapely_box
from pism_terra.vector import get_glacier_from_rgi_id
-from pism_terra.workflow import check_xr_lazy
+from pism_terra.workflow import (
+ check_rio,
+ check_xr_lazy,
+ drop_geotransform_attr,
+ stamp_grid_mapping,
+)
+
+# RGI o1 codes for which ITS_LIVE v2.1 publishes a per-region COG. 13/15/16 are
+# absent because they're merged into the High Mountain Asia (14) mosaic.
+_ITS_LIVE_REGION_CODES: tuple[str, ...] = (
+ "01",
+ "02",
+ "03",
+ "04",
+ "05",
+ "06",
+ "07",
+ "08",
+ "09",
+ "10",
+ "11",
+ "12",
+ "14",
+ "17",
+ "18",
+ "19",
+)
+
+
+@lru_cache(maxsize=None)
+def _its_live_region_footprint(region_code: str):
+ """
+ Return ``(crs, bounds_polygon)`` for an ITS_LIVE per-region COG.
+
+ Opens the COG via ``/vsicurl/`` and reads only the header (no data
+ transfer beyond a few KB). Cached so each region is probed once per
+ process.
+
+ Parameters
+ ----------
+ region_code : str
+ Two-digit RGI o1 code (e.g. ``"01"`` for Alaska).
+
+ Returns
+ -------
+ tuple
+ ``(rasterio.crs.CRS, shapely.geometry.Polygon)`` — the COG's native
+ CRS and its full extent as a rectangular polygon in that CRS.
+ """
+ url = (
+ "/vsicurl/https://its-live-data.s3.amazonaws.com/velocity_mosaic/v2.1/static/cog/"
+ f"ITS_LIVE_velocity_120m_RGI{region_code}A_0000_V02.1_v.tif"
+ )
+ with rasterio.open(url) as src:
+ b = src.bounds
+ return src.crs, _shapely_box(b.left, b.bottom, b.right, b.top)
+
+
+def region_code_from_bounds(bounds: tuple[float, float, float, float], crs: str) -> str:
+ """
+ Return the RGI region code whose ITS_LIVE COG contains ``bounds``.
+
+ Probes each published v2.1 per-region COG header on S3 (cached per
+ process), reprojects ``bounds`` into the COG's CRS, and returns the
+ first region whose footprint fully contains them.
+
+ Parameters
+ ----------
+ bounds : tuple of float
+ ``(minx, miny, maxx, maxy)`` in the CRS given by ``crs``.
+ crs : str
+ CRS of ``bounds`` (anything pyproj accepts: EPSG code, WKT, …).
+
+ Returns
+ -------
+ str
+ Two-digit RGI o1 region code (e.g. ``"01"``).
+
+ Raises
+ ------
+ ValueError
+ If no published region COG fully contains ``bounds``.
+ """
+ minx, miny, maxx, maxy = bounds
+ for code in _ITS_LIVE_REGION_CODES:
+ cog_crs, cog_poly = _its_live_region_footprint(code)
+ t = Transformer.from_crs(crs, cog_crs, always_xy=True)
+ ub = t.transform_bounds(minx, miny, maxx, maxy)
+ if cog_poly.contains(_shapely_box(*ub)):
+ return code
+ raise ValueError(
+ f"No ITS_LIVE per-region COG fully contains bounds {bounds} (crs={crs}). "
+ "Region may straddle two regions, or fall outside published coverage."
+ )
def get_velocities_by_bounds(
bounds: tuple[float, float, float, float],
product_name: str = "its_live",
+ src_crs: str | None = None,
) -> xr.Dataset:
"""
Retrieve and subset a velocity product over a specified geographic bounding box.
@@ -48,11 +148,13 @@ def get_velocities_by_bounds(
Parameters
----------
bounds : tuple of float
- Geographic bounding box in EPSG:4326 coordinates, formatted as
- (minx, miny, maxx, maxy).
+ Bounding box ``(minx, miny, maxx, maxy)`` in the CRS given by ``src_crs``.
product_name : {"its_live"}, optional
The name of the velocity product to query. Currently only "its_live" is supported.
Default is "its_live".
+ src_crs : str or None, optional
+ CRS of ``bounds`` (e.g., ``"EPSG:3413"``). If ``None``, ``"EPSG:4326"``
+ (longitude/latitude) is assumed.
Returns
-------
@@ -71,12 +173,14 @@ def get_velocities_by_bounds(
- This function currently only supports the ITS_LIVE global velocity mosaic.
"""
- # Define source CRS
- src_crs = "EPSG:4326"
+ # Define source CRS if not given.
+ if src_crs is None:
+ src_crs = "EPSG:4326"
# Load dataset
if product_name == "its_live":
- ds = get_itslive_velocities()
+ region_code = region_code_from_bounds(bounds, crs=src_crs)
+ ds = get_itslive_velocities_by_region_code(region_code)
else:
raise NotImplementedError(f"Velocity product '{product_name}' is not supported.")
@@ -86,14 +190,15 @@ def get_velocities_by_bounds(
# Transform bounds to destination CRS
transformer = Transformer.from_crs(src_crs, dst_crs, always_xy=True)
bbox_out = transformer.transform_bounds(*bounds)
-
# Clip dataset
subset = ds.rio.clip_box(minx=bbox_out[0], miny=bbox_out[1], maxx=bbox_out[2], maxy=bbox_out[3])
return subset
-def get_itslive_velocities(components: list[str] = ["v", "vx", "vy", "vx_error", "vy_error"]) -> xr.Dataset:
+def get_itslive_velocities_by_region_code(
+ region_code: str, components: list[str] = ["v", "vx", "vy", "vx_error", "vy_error", "landice"]
+) -> xr.Dataset:
"""
Load the global ITS_LIVE surface velocity mosaic as an xarray dataset.
@@ -102,12 +207,16 @@ def get_itslive_velocities(components: list[str] = ["v", "vx", "vy", "vx_error",
Parameters
----------
+ region_code : str
+ Two-digit RGI o1 code (e.g. ``"01"`` for Alaska).
components : list of str, optional
List of velocity components to load. Valid entries include:
- "v": velocity magnitude
- "vx": x-component of velocity
- "vy": y-component of velocity
- Defaults to ["v", "vx", "vy"].
+ - "vx_error": x-component error
+ - "vy_error": y-component error
+ Defaults to ["v", "vx", "vy", "vx_error", "vy_error", "landice"].
Returns
-------
@@ -122,50 +231,66 @@ def get_itslive_velocities(components: list[str] = ["v", "vx", "vy", "vx_error",
- Missing values are represented using a mask (`masked=True`).
- Coordinate metadata and CRS are read from the VRT headers.
"""
+ # Per-component CF metadata. Everything in this VRT family is m/yr except
+ # the integer ``landice`` mask.
+ component_attrs = {
+ "v": {"units": "m year^-1", "long_name": "ice speed"},
+ "vx": {"units": "m year^-1", "long_name": "x component of ice velocity"},
+ "vy": {"units": "m year^-1", "long_name": "y component of ice velocity"},
+ "vx_error": {"units": "m year^-1", "long_name": "x component error"},
+ "vy_error": {"units": "m year^-1", "long_name": "y component error"},
+ "landice": {"units": "1", "long_name": "land ice mask (1=ice)"},
+ }
dss = []
for c in components:
- url = f"""https://its-live-data.s3-us-west-2.amazonaws.com/velocity_mosaic/v2/static/cog_global/ITS_LIVE_velocity_120m_0000_v02_{c}.vrt"""
- ds = (
+ url = (
+ "https://its-live-data.s3.amazonaws.com/velocity_mosaic/v2.1/static/cog/"
+ f"ITS_LIVE_velocity_120m_RGI{region_code}A_0000_V02.1_{c}.tif"
+ )
+ _ds = (
rxr.open_rasterio(url, parse_coordinates=True, chunks={"x": 1024, "y": 1024}, masked=True)
.isel(band=0)
.drop_vars("band")
)
- ds.name = c
- dss.append(ds)
+ _ds.name = c
+ # Drop junk band-level attrs that rioxarray surfaces from the COG.
+ for k in ("scale_factor", "add_offset", "AREA_OR_POINT", "_FillValue"):
+ _ds.attrs.pop(k, None)
+ _ds.attrs.update(component_attrs.get(c, {}))
+ dss.append(_ds)
- return xr.merge(dss)
+ ds = xr.merge(dss, compat="no_conflicts")
+ return ds
-def glacier_velocities_from_rgi_id(
- rgi_id: str,
- rgi: gpd.GeoDataFrame | str | Path = "rgi/rgi.gpkg",
+
+def glacier_velocities_from_grid(
+ target_grid: xr.Dataset,
+ geometries: collections.abc.Iterable,
product_name: str = "its_live",
- buffer_distance: float = 2000.0,
path: Path | str = "tmp.nc",
force_overwrite: bool = False,
) -> xr.Dataset:
"""
Generate observed glacier surface velocities for a glacier by RGI ID.
- Extracts the glacier geometry, builds a buffered extent, fetches a velocity
+ Extracts the glacier geometry, builds an extent, fetches a velocity
product (e.g., ITS_LIVE) over that region, clips it to the glacier outline,
and returns the result as an xarray dataset. A cached NetCDF at ``path`` is
reused unless ``force_overwrite=True``.
Parameters
----------
- rgi_id : str
- Glacier identifier (e.g., ``"RGI2000-v7.0-C-06-00014"``).
- rgi : geopandas.GeoDataFrame or str or pathlib.Path, default ``"rgi/rgi.gpkg"``
- In-memory RGI table or a path to a GeoPackage/shape readable by
- :func:`geopandas.read_file`. Must contain the feature with ``rgi_id``
- and an ``epsg`` column for the glacier CRS.
+ target_grid : xarray.Dataset
+ Target grid dataset whose ``x``/``y`` extent (in the grid's projected CRS,
+ as recorded in ``spatial_ref``) defines the velocity query region. The
+ velocity product is reprojected/aligned to this grid.
+ geometries : iterable of shapely geometries
+ Glacier outline(s) in ``target_grid``'s CRS. Used to clip the velocity
+ dataset to the glacier footprint.
product_name : str, default ``"its_live"``
Velocity product to retrieve (e.g., ``"its_live"``). Passed to
:func:`get_velocities_by_bounds`.
- buffer_distance : float, default ``2000.0``
- Buffer (meters) applied to the glacier polygon in its projected CRS to
- form the query extent for the velocity product.
path : str or pathlib.Path, default ``"tmp.nc"``
Cache file for the clipped velocity dataset. When present and valid
(per :func:`check_xr_lazy`), it is opened instead of re-downloading.
@@ -179,64 +304,251 @@ def glacier_velocities_from_rgi_id(
on the source product but typically include components (e.g., ``u``,
``v`` or ``vx``, ``vy``) and possibly speed (e.g., ``v``). CRS is
recorded via :mod:`rioxarray`.
-
- Raises
- ------
- FileNotFoundError
- If the provided RGI path does not exist.
- ValueError
- If ``rgi_id`` is missing from the RGI layer or CRS info is invalid.
- Exception
- Propagated I/O, reprojection, or decoding errors from helper functions.
-
- See Also
- --------
- get_glacier_from_rgi_id
- Look up the glacier feature/CRS from the RGI table.
- get_velocities_by_bounds
- Fetch velocity data for a geographic bounding box.
- check_xr_lazy
- Lightweight validity check for an on-disk xarray dataset.
-
- Notes
- -----
- - Buffering is performed in the glacier’s projected CRS (meters) and the
- buffered geometry is reprojected to WGS84 only to compute geographic
- bounds for the velocity query.
- - On cache reuse, the code sets the dataset CRS to the glacier CRS; on a
- fresh download, CRS comes from the source product. If you require a
- specific target CRS, reproject with ``.rio.reproject_match(...)``.
"""
print("")
print("Generate Velocity Observations")
- print("-" * 80)
-
- crs = "EPSG:3857"
+ print("-" * 120)
- if isinstance(rgi, str | Path):
- rgi = gpd.read_file(rgi)
- glacier = get_glacier_from_rgi_id(rgi, rgi_id)
- glacier_series = glacier.iloc[0]
- dst_crs = glacier_series["epsg"]
+ EPS = 10.0
if (not check_xr_lazy(path)) or force_overwrite:
path = Path(path)
path.unlink(missing_ok=True)
- glacier_projected = glacier.to_crs(dst_crs)
- geometry_buffered_projected = glacier_projected.geometry.buffer(buffer_distance)
- geometry_buffered_geoid = geometry_buffered_projected.to_crs("EPSG:4326").iloc[0]
+ xs = [float(target_grid.x.values[0]), float(target_grid.x.values[-1])]
+ ys = [float(target_grid.y.values[0]), float(target_grid.y.values[-1])]
+ bounds = (min(xs), min(ys), max(xs), max(ys))
+ mapping_var = target_grid.rio.grid_mapping
+ dst_crs = target_grid[mapping_var].attrs["crs_wkt"]
+ t_geo = Transformer.from_crs(dst_crs, "EPSG:4326", always_xy=True)
+ geo_bounds = t_geo.transform_bounds(*bounds)
+ # Pad the geographic bbox so the clipped ITS_LIVE region fully covers
+ # every target-grid point after round-tripping through 4326 → 3413.
+ lon_pad = 0.25
+ lat_pad = 0.1
+ padded = (
+ geo_bounds[0] - lon_pad,
+ geo_bounds[1] - lat_pad,
+ geo_bounds[2] + lon_pad,
+ geo_bounds[3] + lat_pad,
+ )
+ ds = get_velocities_by_bounds(padded, product_name=product_name)
+
+ # The interpolator is built on ITS_LIVE's native coordinates, so the
+ # intermediate frame must match. The finite-difference round-trip
+ # below recovers vector components aligned with target_grid's axes
+ # (handling any rotation between the two CRSs).
+ src_crs = ds.rio.crs
+ t = Transformer.from_crs(dst_crs, src_crs, always_xy=True)
+ t_inv = Transformer.from_crs(src_crs, dst_crs, always_xy=True)
+ # Define DEM grid
+ X, Y = np.meshgrid(target_grid.x, target_grid.y)
+
+ # Project to ITSLive grid
+ X_, Y_ = t.transform(X, Y)
+
+ # Build ITSLive interpolants. ``bounds_error=False`` makes points that
+ # land just outside the clipped tile (or in ITS_LIVE nodata regions)
+ # fall back to NaN instead of raising; they're zeroed later.
+ interpolator_vx = RegularGridInterpolator(
+ (ds.y, ds.x), ds.vx.values.squeeze(), bounds_error=False, fill_value=np.nan
+ )
+ interpolator_vy = RegularGridInterpolator(
+ (ds.y, ds.x), ds.vy.values.squeeze(), bounds_error=False, fill_value=np.nan
+ )
+
+ # Interpolate dem grid points
+ vx_pts = interpolator_vx((Y_, X_))
+ vy_pts = interpolator_vy((Y_, X_))
+
+ # Finite difference displacement
+ X_plus = X_ + EPS * vx_pts
+ Y_plus = Y_ + EPS * vy_pts
- bounds = geometry_buffered_geoid.bounds
+ X_minus = X_ - EPS * vx_pts
+ Y_minus = Y_ - EPS * vy_pts
- ds = get_velocities_by_bounds(bounds, product_name=product_name)
- glacier_projected = glacier.to_crs(crs)
- ds_clipped = ds.rio.clip(glacier_projected.geometry)
+ # Transform displaced points back to project grid
+ X0_plus, Y0_plus = t_inv.transform(X_plus, Y_plus)
+ X0_minus, Y0_minus = t_inv.transform(X_minus, Y_minus)
+
+ # Calculate velocities
+ vx = (X0_plus - X0_minus) / (2 * EPS)
+ vy = (Y0_plus - Y0_minus) / (2 * EPS)
+
+ # Reproject to the glacier's target CRS to match the PISM grid
+ ds_clipped = ds.rio.reproject_match(target_grid, resampling=Resampling.bilinear).rio.clip(
+ geometries, drop=False
+ )
+ # Snapshot ITS_LIVE's coverage from the reprojected/clipped ``v`` before
+ # we overwrite ``vx``/``vy``/``v`` with the finite-difference field
+ # (which is finite everywhere on the mesh and so erases the original
+ # NaN pattern that the masks below depend on).
+ v_missing = ds_clipped["v"].isnull().copy()
+ ds_clipped["vx"].values = vx
+ ds_clipped["vy"].values = vy
+
+ # Zero out the velocity fields (and their per-component errors) off
+ # ice. ITS_LIVE's ``landice`` is 1 over glacier ice and 0 elsewhere,
+ # but the COG declares nodata=0 so reading with ``masked=True`` turns
+ # off-ice cells into NaN. Test for on-ice (== 1) so both 0 and NaN
+ # count as off-ice; ``where(cond, 0)`` keeps the value when cond is
+ # true and writes 0 otherwise.
+ if "landice" in ds_clipped:
+ on_ice = ds_clipped["landice"] == 1
+ for name in ("vx", "vy", "vx_error", "vy_error"):
+ if name in ds_clipped:
+ ds_clipped[name] = ds_clipped[name].where(on_ice, 0)
+
+ ds_clipped["v"].values = (ds_clipped["vx"].values ** 2 + ds_clipped["vy"].values ** 2) ** 0.5
+ ds_clipped["u_observed"] = ds_clipped["vx"].fillna(0)
+ ds_clipped["v_observed"] = ds_clipped["vy"].fillna(0)
+
+ ds_clipped["zeta_fixed_mask"] = xr.where(v_missing, 1, 0).fillna(0).astype(int)
+ ds_clipped["vel_misfit_weight"] = xr.where(v_missing, 0, 1).fillna(0).astype(int)
+ ds_clipped["vel_misfit_weight"].attrs.update(
+ {"units": "1", "long_name": "misfit weight (1=trust obs, 0=ignore)"}
+ )
+ ds_clipped["zeta_fixed_mask"].attrs.update({"units": "1", "long_name": "fixed zeta mask (1=no obs, fix prior)"})
+
+ # Spatially constant basal-yield-stress prior for the inversion.
+ ds_clipped["tauc_prior"] = xr.full_like(ds_clipped["v_observed"], 1.4e5)
+ ds_clipped["tauc_prior"].attrs = {
+ "units": "Pa",
+ "long_name": "prior basal yield stress (constant)",
+ }
+
+ # Stamp CF metadata on the projected x/y coords (lost across some
+ # rioxarray ops) and suppress the default ``_FillValue=NaN`` netCDF4
+ # writes onto coordinate variables.
+ ds_clipped["x"].attrs.update(
+ {
+ "standard_name": "projection_x_coordinate",
+ "long_name": "x coordinate of projection",
+ "units": "m",
+ "axis": "X",
+ }
+ )
+ ds_clipped["y"].attrs.update(
+ {
+ "standard_name": "projection_y_coordinate",
+ "long_name": "y coordinate of projection",
+ "units": "m",
+ "axis": "Y",
+ }
+ )
+ ds_clipped["x"].encoding["_FillValue"] = None
+ ds_clipped["y"].encoding["_FillValue"] = None
+
+ # Strip junk band metadata that rioxarray inherited from the source
+ # COGs and propagated through reprojection.
+ for name in ds_clipped.data_vars:
+ for k in ("scale_factor", "add_offset", "AREA_OR_POINT"):
+ ds_clipped[name].attrs.pop(k, None)
+ ds_clipped[name].encoding.pop(k, None)
+
+ # Re-attach the CRS + grid_mapping on every data_var. ``.where`` and
+ # the ``u_observed``/``v_observed`` reconstructions drop the
+ # ``grid_mapping`` encoding key, so only untouched vars (``v``,
+ # ``landice``) would otherwise carry it through to the written file.
+ mapping_var = target_grid.rio.grid_mapping
+ crs = target_grid[mapping_var].attrs["crs_wkt"]
+ ds_clipped = ds_clipped.rio.write_crs(crs).rio.write_grid_mapping().rio.write_coordinate_system()
+
+ ds_clipped = stamp_grid_mapping(ds_clipped)
+ # Drop the GeoTransform so GDAL/QGIS derive the (top-down) transform from
+ # the ascending y coordinate instead of rendering the raster upside-down.
+ drop_geotransform_attr(ds_clipped)
ds_clipped.to_netcdf(path)
else:
ds_clipped = xr.open_dataset(path)
- ds_clipped.rio.write_crs(crs, inplace=True)
+ mapping_var = target_grid.rio.grid_mapping
+ crs = target_grid[mapping_var].attrs["crs_wkt"]
+ ds_clipped = ds_clipped.rio.write_crs(crs).rio.write_grid_mapping().rio.write_coordinate_system()
return ds_clipped
+
+
+def bathymetry_from_grid(
+ target_grid: xr.Dataset,
+ uri: str,
+ path: Path | str = "tmp.nc",
+ force_overwrite: bool = False,
+) -> xr.DataArray:
+ """
+ Build a glacier-domain bathymetry/elevation field from a cloud raster.
+
+ Opens a remote raster (typically a Cloud Optimized GeoTIFF on S3 referenced
+ via ``/vsis3/`` or ``/vsicurl/``), clips it to the geographic bounds of
+ ``target_grid``, reprojects to ``target_grid``'s CRS/extent, and returns the
+ result as a DataArray. A cached NetCDF at ``path`` is reused unless
+ ``force_overwrite=True``.
+
+ Parameters
+ ----------
+ target_grid : xarray.Dataset
+ Target grid dataset whose ``x``/``y`` extent (in the grid's projected
+ CRS, as recorded in ``spatial_ref``) defines the query region. The
+ bathymetry raster is reprojected/aligned to this grid.
+ uri : str
+ Path/URI of the source raster. Local paths and GDAL VSI URIs are both
+ accepted (e.g., ``"/vsis3/bucket/key.tif"`` or
+ ``"/vsicurl/https://.../bathymetry.tif"``).
+ path : str or pathlib.Path, default ``"tmp.nc"``
+ Cache file for the clipped/reprojected output. When present and valid
+ (per :func:`check_rio`), it is opened instead of re-fetching.
+ force_overwrite : bool, default ``False``
+ If ``True``, ignore any existing cache at ``path`` and regenerate.
+
+ Returns
+ -------
+ xarray.DataArray
+ Bathymetry/elevation values (float32, meters) on ``target_grid`` with
+ CRS attached via :mod:`rioxarray`.
+ """
+
+ print("")
+ print("Generate Bathymetry")
+ print("-" * 120)
+
+ if (not check_rio(path)) or force_overwrite:
+
+ path = Path(path)
+ path.unlink(missing_ok=True)
+
+ bounds = [target_grid.x.values[0], target_grid.y.values[0], target_grid.x.values[-1], target_grid.y.values[-1]]
+ mapping_var = target_grid.rio.grid_mapping
+ dst_crs = target_grid[mapping_var].attrs["crs_wkt"]
+ t = Transformer.from_crs(dst_crs, "EPSG:4326", always_xy=True)
+ geo_bounds = t.transform_bounds(*bounds)
+
+ da = rxr.open_rasterio(uri, masked=True, chunks={"x": 1024, "y": 1024}).squeeze()
+ # GEBCO is a global EPSG:4326 COG; if the target grid wraps the
+ # antimeridian (pyproj returns xmin > xmax, e.g. RGI region 01),
+ # clip the two 4326 halves separately, reproject each into the
+ # projected target grid (continuous across the seam), and coalesce.
+ if geo_bounds[0] > geo_bounds[2]:
+ west = da.rio.clip_box(geo_bounds[0], geo_bounds[1], 180.0, geo_bounds[3], crs=da.rio.crs)
+ east = da.rio.clip_box(-180.0, geo_bounds[1], geo_bounds[2], geo_bounds[3], crs=da.rio.crs)
+ west_reproj = west.rio.reproject_match(target_grid, resampling=Resampling.bilinear)
+ east_reproj = east.rio.reproject_match(target_grid, resampling=Resampling.bilinear)
+ out = west_reproj.fillna(east_reproj).astype("float32")
+ else:
+ sub = da.rio.clip_box(*geo_bounds, crs=da.rio.crs)
+ out = sub.rio.reproject_match(target_grid, resampling=Resampling.bilinear).astype("float32")
+ out.encoding = {} # drop stale int16 dtype/fill from the source COG
+ out = out.rio.write_crs(dst_crs).rio.write_grid_mapping()
+ out.name = "bathymetry"
+ # Strip stale per-band attrs that confuse xarray on re-read
+ for k in ("scale_factor", "add_offset", "AREA_OR_POINT"):
+ out.attrs.pop(k, None)
+ # Drop the GeoTransform so GDAL/QGIS derive the (top-down) transform from
+ # the ascending y coordinate instead of rendering the raster upside-down.
+ drop_geotransform_attr(out)
+ out.to_netcdf(path)
+
+ else:
+ out = xr.open_dataset(path)["bathymetry"]
+ return out
diff --git a/pism_terra/glacier/postprocess.py b/pism_terra/glacier/postprocess.py
index 48f35bb..f564ce7 100644
--- a/pism_terra/glacier/postprocess.py
+++ b/pism_terra/glacier/postprocess.py
@@ -22,9 +22,12 @@
"""
import json
+import logging
import time
+import warnings
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from pathlib import Path
+from typing import Literal
import cf_xarray
import dask
@@ -32,99 +35,159 @@
import rioxarray
import toml
import xarray as xr
-from dask.diagnostics import ProgressBar
from dask.distributed import Client, progress
from pyfiglet import Figlet
+from tqdm import tqdm
+
+from pism_terra.log import setup_logging
xr.set_options(keep_attrs=True)
+warnings.filterwarnings("ignore", message="invalid value encountered in cast", category=RuntimeWarning)
+warnings.filterwarnings("ignore", message="pkg_resources is deprecated", category=UserWarning)
+
+logger = logging.getLogger(__name__)
-def process_file(infile: str | Path, rgi_file: str | Path):
+def process_file(
+ infile: str | Path,
+ outline_file: str | Path,
+ rgi_type: Literal["G", "C"],
+ client: Client,
+ column: str = "rgi_id",
+):
"""
- Clip a NetCDF dataset to the glacier geometry defined in an RGI file.
+ Clip a NetCDF dataset to glacier outlines and write per-outline scalar sums.
- This function reads a NetCDF file containing geospatial data and clips it to the
- geometry defined in a glacier outline file (e.g., RGI shapefile). The clipped dataset
- is saved to a new NetCDF file prefixed with "clipped_".
+ Reads ``infile``, reprojects the geometries in ``outline_file`` to the
+ dataset's CRS, and clips the dataset to each outline. For every outline the
+ spatial variables are summed over ``x``/``y`` (with the outline ``area``
+ attached) and stacked along an ``rgi_id`` dimension. The result is written
+ to ``/processed_scalar/fldsum__.nc``, where
+ ``output_root`` is the grandparent of ``infile`` (``infile.parent.parent``)
+ and ```` is the input filename.
Parameters
----------
infile : str or Path
Path to the NetCDF file to be clipped. Must contain x/y spatial dimensions.
- rgi_file : str or Path
- Path to the RGI glacier outline file (e.g., GeoPackage or shapefile) that defines
- the geometry to clip the dataset to. Must include an `epsg` column to define the CRS.
+ outline_file : str or Path
+ Path to the glacier outline file (e.g., GeoPackage or shapefile) whose
+ geometries the dataset is clipped to.
+ rgi_type : {"G", "C"}
+ RGI outline type being processed — glacier ("G") or glacier complex
+ ("C"). Used as a prefix in the output filename (``fldsum__...``).
+ client : dask.distributed.Client
+ Dask client used to persist the dataset before clipping.
+ column : str, default "rgi_id"
+ Name of the column in ``outline_file`` used to label each clipped
+ outline along the output ``rgi_id`` dimension.
"""
infile = Path(infile)
infile_name = infile.name
- infile_path = infile.parent
- clipped_file = infile_path / Path("clipped_" + infile_name)
- speed_clipped_file = infile_path / Path("clipped_speed_" + infile_name)
- scalar_file = infile_path / Path("fldsum_" + infile_name)
+ output_root = infile.parent.parent
+ scalar_dir = output_root / "processed_scalar"
+ scalar_dir.mkdir(parents=True, exist_ok=True)
+ scalar_file = scalar_dir / Path(f"fldsum_{rgi_type}_" + infile_name)
- rgi = gpd.read_file(rgi_file)
- crs = rgi.iloc[0]["epsg"]
- rgi_projected = rgi.to_crs(crs)
- geometry = rgi_projected.geometry
+ outline = gpd.read_file(outline_file)
start = time.time()
-
- ds = (
- xr.open_dataset(
- infile,
- decode_times=False,
- decode_timedelta=False,
- chunks="auto",
- engine="h5netcdf",
- )
- .drop_vars("time_bounds", errors="ignore")
- .rio.set_spatial_dims(x_dim="x", y_dim="y")
+ time_coder = xr.coders.CFDatetimeCoder(use_cftime=False)
+
+ ds = xr.open_dataset(
+ infile,
+ decode_timedelta=False,
+ decode_times=False,
+ chunks="auto",
+ engine="netcdf4",
)
+ mapping_var = ds.rio.grid_mapping
+ dst_crs = ds[mapping_var].attrs["crs_wkt"]
+
+ outline = outline.to_crs(dst_crs)
+
+ # Spatial bounds vars (``x_bnds``/``y_bnds``) reference the pre-clip
+ # x/y sizes and would inject dangling dimensions back into the output
+ # after merge — h5netcdf serializes those as duplicate "x" dims. Drop
+ # them before splitting; PISM doesn't require them on the clipped output.
+ ds = ds.drop_vars(["x_bnds", "x_bounds", "y_bnds", "y_bounds", "mapping", "spatial_ref"], errors="ignore")
+
+ # Separate variables that lack BOTH spatial (x, y) dimensions, as
+ # rio.clip cannot handle them. Use ``and`` so that vars carrying only
+ # one spatial dim (rare, but possible) still go down the spatial path.
+ non_spatial_vars = [var for var in ds.data_vars if "x" not in ds[var].dims and "y" not in ds[var].dims]
+ ds_non_spatial = ds[non_spatial_vars]
+ ds = ds.drop_vars(non_spatial_vars).rio.write_crs(dst_crs).rio.set_spatial_dims(x_dim="x", y_dim="y")
+ ds = client.persist(ds)
+ progress(ds)
+
+ comp = {"zlib": True, "complevel": 2}
+
+ dss = []
+ for _, row in tqdm(outline.iterrows(), total=len(outline), desc="Clipping outlines"):
+ ds_clipped = ds.rio.clip([row.geometry], drop=False)
+ ds_sum = ds_clipped.sum(dim=["y", "x"]).compute()
+ ds_sum["area"] = row.geometry.area
+ ds_sum["area"].attrs.update({"units": "m^2"})
+ dss.append(ds_sum.expand_dims({"RGIid": [row[column]]}))
+
+ scalar = xr.concat(dss, dim="RGIid")
+
+ logger.info("Writing %s", scalar_file)
+ # Keep non-spatial vars (e.g. pism_config)
+ extra_vars = [v for v in ds_non_spatial.data_vars if "time" not in ds_non_spatial[v].dims]
+ if extra_vars:
+ scalar = xr.merge([scalar, ds_non_spatial[extra_vars].compute()])
+ encoding_scalar = {var: comp for var in scalar.data_vars}
+ scalar.to_netcdf(scalar_file, encoding=encoding_scalar, engine="netcdf4")
- ds = ds.rio.write_crs(crs, inplace=False)
- ds_clipped = ds.rio.clip(geometry, drop=False)
- ds_clipped.to_netcdf(clipped_file)
end = time.time()
time_elapsed = end - start
- print(f"Time elapsed for postprocessing: {time_elapsed:.0f}s")
- pism_config = ds["pism_config"]
- ds_scalar = ds_clipped.drop_vars(["pism_config"], errors="ignore").sum(dim=["x", "y"])
- ds_scalar = xr.merge([ds_scalar, pism_config])
- ds_scalar.to_netcdf(scalar_file)
+ logger.info("Time elapsed for %s: %.0fs", infile_name, time_elapsed)
-def postprocess_glacier(config_file: str | Path):
+def postprocess_glacier(config_file: str | Path, rgi_type: Literal["G", "C"], n_workers: int = 4):
"""
- Configure and print a PISM model run command for a glacier.
+ Postprocess a PISM glacier ``spatial`` output for one RGI outline type.
- This function reads glacier metadata from a CSV file and simulation settings
- from a TOML configuration file, then builds and prints a full PISM command-line
- string for executing a model run. It sets up output directories and constructs
- appropriate output filenames.
+ Reads simulation paths from a TOML run configuration and clips the
+ ``spatial`` output NetCDF to the requested RGI outline (glacier ``"G"`` or
+ complex ``"C"``) using a Dask client, writing per-outline scalar sums via
+ :func:`process_file`.
Parameters
----------
config_file : str or Path
- Path to a TOML file containing PISM run configuration, including time,
- energy model, stress balance model, and reporting options.
+ Path to a TOML file containing the PISM run configuration. The
+ ``[rgi]`` table provides the outline paths (``outline_g``/``outline_c``,
+ with a legacy ``outline`` fallback) and ``[output]`` the file paths.
+ rgi_type : {"G", "C"}
+ RGI outline type to process. Selects ``outline_`` from the
+ config's ``[rgi]`` table.
+ n_workers : int, optional
+ Number of Dask workers, by default 4.
"""
config_toml = toml.load(config_file)
config = json.loads(json.dumps(config_toml))
start = time.time()
- rgi_file = config["rgi"]["outline"]
+ rgi = config["rgi"]
+ outline_file = rgi.get(f"outline_{rgi_type.lower()}", rgi.get("outline"))
+
+ client = Client(n_workers=n_workers, threads_per_worker=1)
+ logger.info("Dask dashboard: %s", client.dashboard_link)
- for o in ["spatial", "state"]:
+ for o in ["spatial"]:
s_file = Path(config["output"][o])
- print(s_file)
- with ProgressBar():
- process_file(s_file, rgi_file)
+ process_file(s_file, outline_file, rgi_type, client)
+
+ client.close()
end = time.time()
time_elapsed = end - start
- print(f"Time elapsed {time_elapsed:.0f}s")
+ logger.info("Time elapsed %.0fs", time_elapsed)
def main():
@@ -135,6 +198,12 @@ def main():
# set up the option parser
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.description = "Postprocess RGI Glacier."
+ parser.add_argument(
+ "--ntasks",
+ help="Sets number of tasks.",
+ type=int,
+ default=4,
+ )
parser.add_argument(
"RUN_FILE",
help="CONFIG TOML.",
@@ -143,8 +212,13 @@ def main():
options, unknown = parser.parse_known_args()
config_file = options.RUN_FILE[0]
+ ntasks = options.ntasks
+
+ config_path = Path(config_file).resolve().parent
+ setup_logging(config_path / "postprocess.log")
- postprocess_glacier(config_file)
+ postprocess_glacier(config_file, "C", n_workers=ntasks)
+ postprocess_glacier(config_file, "G", n_workers=ntasks)
if __name__ == "__main__":
diff --git a/pism_terra/glacier/prepare.py b/pism_terra/glacier/prepare.py
index 8adf5e7..f601482 100644
--- a/pism_terra/glacier/prepare.py
+++ b/pism_terra/glacier/prepare.py
@@ -18,7 +18,7 @@
# pylint: disable=too-many-positional-arguments,unused-import,broad-exception-caught
"""
-Prepare RGI data sets.
+Prepare RGI and S4F data sets.
"""
import logging
@@ -51,10 +51,27 @@
from rasterio.warp import Resampling, calculate_default_transform, reproject
from tqdm.auto import tqdm
-from pism_terra.download import download_archive, download_file, extract_archive
-from pism_terra.glacier.climate import convert_many_tifs_concurrent
-from pism_terra.glacier.ice_thickness import prepare_ice_thickness_maffezzoli
+from pism_terra.download import (
+ download_archive,
+ download_file,
+ download_gebco,
+ extract_archive,
+)
+from pism_terra.glacier.climate import (
+ convert_many_tifs_concurrent,
+ prepare_carra2,
+ prepare_carra2_for_group,
+ prepare_glaciermip4,
+ prepare_snap,
+)
+from pism_terra.glacier.ice_thickness import (
+ prepare_ice_thickness_frank,
+ prepare_ice_thickness_maffezzoli,
+)
from pism_terra.glacier.rgi import prepare_rgi
+from pism_terra.heatflux import prepare_heatflux_lucazeau
+from pism_terra.log import setup_logging
+from pism_terra.prepare_select import add_include_argument, select_datasets
from pism_terra.vector import glaciers_in_complex
from pism_terra.workflow import check_xr_lazy
@@ -62,10 +79,32 @@
logger = logging.getLogger(__name__)
+# Datasets each prepare command can process, in execution order. Used for the
+# ``--include`` selector and its help text.
+RGI_DATASETS = [
+ "rgi",
+ "heatflux_lucazeau",
+ "ice_thickness_frank",
+ "ice_thickness_maffezzoli",
+ "gebco",
+ "snap",
+ "carra2",
+]
+S4F_DATASETS = [
+ "rgi",
+ "heatflux_lucazeau",
+ "ice_thickness_frank",
+ "ice_thickness_maffezzoli",
+ "gebco",
+ "glaciermip4",
+ "snap",
+ "carra2",
+]
-def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
+
+def s4f(argv: Sequence[str] | None = None) -> dict[str, Any]:
"""
- Prepare RGI input data sets.
+ Prepare S4F glacier input data sets.
This function is the programmatic entry point. It parses command-line style
arguments, creates the target grid, downloads and processes observation data,
@@ -82,10 +121,266 @@ def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
Returns
-------
dict[str, Any]
- Results dictionary containing:
+ Mapping returned by :func:`prepare_rgi` (e.g. ``"rgi_complexes"``
+ and ``"rgi_glaciers"`` paths).
+ """
+
+ parser = ArgumentParser()
+ parser.add_argument(
+ "--force-overwrite",
+ help="Force downloading all files.",
+ action="store_true",
+ default=False,
+ )
+ parser.add_argument(
+ "--ntasks",
+ help="Parallel tasks.",
+ type=int,
+ default=8,
+ )
+ add_include_argument(parser, S4F_DATASETS)
+ parser.add_argument("CONFIG_FILE", nargs=1)
+ parser.add_argument("OUTPUT_PATH", nargs=1)
+ parser.add_argument("GLACIER_FILES", nargs="*")
+ args = parser.parse_args(list(argv) if argv is not None else None)
+
+ config_file = args.CONFIG_FILE[0]
+ glacier_files = args.GLACIER_FILES
+ force_overwrite = args.force_overwrite
+ ntasks = args.ntasks
+ output_path = Path(args.OUTPUT_PATH[0])
+ output_path.mkdir(parents=True, exist_ok=True)
+
+ setup_logging(output_path / "prepare.log")
+
+ selected = select_datasets(args.include, S4F_DATASETS)
+
+ f = Figlet(font="standard")
+ banner = f.renderText("pism-terra")
+ logger.info("=" * 120)
+ logger.info("\n%s", banner)
+ logger.info("=" * 120)
+ logger.info("Preparing S4F data")
+ logger.info("-" * 120)
+
+ config = toml.loads(Path(config_file).read_text("utf-8"))
+ regions = pd.DataFrame.from_dict(config["regions"], orient="index")
+ regions["region"] = regions.index.astype(str).str.zfill(2) + "_" + regions["name"]
+
+ glacier_groups: dict[str, pd.DataFrame] = {} # aggregated-name -> CSV rows
+ if len(glacier_files) > 0:
+ for glacier_file in glacier_files:
+ p = Path(glacier_file)
+ # "S4F_target_AK_RGI_id.csv" -> "S4F_AK"; fall back to the file stem.
+ m = re.match(r"^(?P.+?)_target_(?P.+?)_RGI_id$", p.stem)
+ name = f"{m['prefix']}_{m['region']}" if m else p.stem
+ glacier_groups[name] = pd.read_csv(p)
+
+ glaciers_csv = pd.concat(glacier_groups.values(), ignore_index=True)
+ glaciers_csv["o1regions"] = glaciers_csv["rgi_id"].str.extract(r"-G-(\d{2})-")
+ o1regions = glaciers_csv["o1regions"].unique().astype(int).astype(str)
+ regions = regions[regions.index.isin(o1regions)]
+ else:
+ glaciers_csv = None
+ # Final outputs land under glacier_path; everything intermediate lives in
+ # staging_path so the user can `rm -rf staging` after a clean run without
+ # losing anything that downstream tools need.
+ glacier_path = output_path / Path("glacier")
+ glacier_path.mkdir(parents=True, exist_ok=True)
+ staging_path = output_path / Path("staging")
+ staging_path.mkdir(parents=True, exist_ok=True)
+
+ # --- RGI ---
+ # The RGI outlines are a dependency of ice thickness (and the CARRA2 group
+ # reprojection); paths are always resolved, ``prepare_rgi`` only runs when
+ # selected, otherwise the cached gpkg files from a previous run are reused.
+ rgi_path = glacier_path / Path("rgi")
+ rgi_path.mkdir(parents=True, exist_ok=True)
+ rgi_staging = staging_path / Path("rgi")
+ rgi_staging.mkdir(parents=True, exist_ok=True)
- - ``"config"``: dict
- The parsed TOML configuration used for processing.
+ rgi_files: dict[str, Any] = {
+ "rgi_complexes": rgi_path / "rgi_c.gpkg",
+ "rgi_glaciers": rgi_path / "rgi_g.gpkg",
+ }
+ if "rgi" in selected:
+ rgi_files = prepare_rgi(
+ regions,
+ glaciers=glaciers_csv,
+ glacier_groups=glacier_groups or None,
+ output_path=rgi_path,
+ extract_path=rgi_staging,
+ force_overwrite=force_overwrite,
+ ntasks=ntasks,
+ )
+
+ # Load the RGI outlines once if any consumer needs them.
+ need_outlines = bool({"ice_thickness_frank", "ice_thickness_maffezzoli"} & set(selected)) or (
+ "carra2" in selected and bool(glacier_groups)
+ )
+ complexes = gpd.read_file(rgi_files["rgi_complexes"]) if need_outlines else None
+ glaciers = gpd.read_file(rgi_files["rgi_glaciers"]) if need_outlines else None
+
+ # --- Ice thickness ---
+ if {"ice_thickness_frank", "ice_thickness_maffezzoli"} & set(selected):
+ ice_thickness_path = glacier_path / Path("ice_thickness")
+ ice_thickness_path.mkdir(parents=True, exist_ok=True)
+ ice_thickness_staging = staging_path / Path("ice_thickness")
+ ice_thickness_staging.mkdir(parents=True, exist_ok=True)
+
+ if "ice_thickness_frank" in selected:
+ frank_path = ice_thickness_path / Path("frank")
+ frank_path.mkdir(parents=True, exist_ok=True)
+ prepare_ice_thickness_frank(
+ regions.index,
+ complexes=complexes,
+ glaciers=glaciers,
+ output_path=frank_path,
+ extract_path=ice_thickness_staging,
+ rgi_extract_path=rgi_staging,
+ force_overwrite=force_overwrite,
+ ntasks=ntasks,
+ )
+
+ if "ice_thickness_maffezzoli" in selected:
+ maffezzoli_path = ice_thickness_path / Path("maffezzoli")
+ maffezzoli_path.mkdir(parents=True, exist_ok=True)
+ prepare_ice_thickness_maffezzoli(
+ regions.index,
+ complexes=complexes,
+ glaciers=glaciers,
+ output_path=maffezzoli_path,
+ extract_path=ice_thickness_staging,
+ force_overwrite=force_overwrite,
+ ntasks=ntasks,
+ )
+
+ # --- GEBCO ---
+ if "gebco" in selected:
+ # Source NetCDF download lands in staging; only the COG goes to glacier/.
+ gebco_path = glacier_path / Path("gebco")
+ gebco_path.mkdir(parents=True, exist_ok=True)
+ gebco_staging = staging_path / Path("gebco")
+ gebco_staging.mkdir(parents=True, exist_ok=True)
+ gebco_nc = download_gebco(target_dir=gebco_staging)
+ cog_gebco_p = gebco_path / Path("bathymetry.tif")
+
+ ds = xr.open_dataset(gebco_nc, chunks={"lat": 1024, "lon": 1024})
+ da = ds["elevation"].rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=False)
+ if da.rio.crs is None:
+ da = da.rio.write_crs("EPSG:4326")
+ predictor = 3 if np.issubdtype(da.dtype, np.floating) else 2
+ da.rio.to_raster(
+ cog_gebco_p,
+ driver="COG",
+ compress="DEFLATE",
+ predictor=predictor,
+ blocksize=512,
+ bigtiff="YES",
+ overview_resampling="AVERAGE",
+ num_threads="ALL_CPUS",
+ )
+
+ # --- Heat flow (Lucazeau 2019) ---
+ if "heatflux_lucazeau" in selected:
+ heatflux_path = glacier_path / Path("heatflux")
+ heatflux_path.mkdir(parents=True, exist_ok=True)
+ heatflux_staging = staging_path / Path("heatflux")
+ heatflux_staging.mkdir(parents=True, exist_ok=True)
+ prepare_heatflux_lucazeau(
+ output_path=heatflux_path,
+ extract_path=heatflux_staging,
+ force_overwrite=force_overwrite,
+ )
+
+ # --- Climate ---
+ if {"glaciermip4", "snap", "carra2"} & set(selected):
+ climate_path = glacier_path / Path("climate")
+ climate_path.mkdir(parents=True, exist_ok=True)
+
+ if "glaciermip4" in selected:
+ glaciermip4_staging = staging_path / Path("glaciermip4")
+ glaciermip4_staging.mkdir(parents=True, exist_ok=True)
+ prepare_glaciermip4(glaciermip4_staging)
+
+ if "snap" in selected:
+ # SNAP/CRU-TS40 monthly climatologies (built under staging, copied to
+ # glacier/climate for upload; one file per 30-year window).
+ snap_staging = staging_path / Path("snap")
+ snap_staging.mkdir(parents=True, exist_ok=True)
+ for snap_file in prepare_snap(snap_staging, force_overwrite=force_overwrite):
+ shutil.copy2(snap_file, climate_path / Path(snap_file).name)
+
+ if "carra2" in selected:
+ # Run the download/merge under staging, then move only the merged
+ # product into glacier/climate. Year-by-year CDS intermediates stay
+ # in staging.
+ carra2_staging = staging_path / Path("carra2")
+ carra2_staging.mkdir(parents=True, exist_ok=True)
+
+ carra2_staging_file = prepare_carra2(carra2_staging)
+ carra2_final = climate_path / Path(carra2_staging_file.name)
+ if carra2_staging_file.is_dir():
+ # Zarr store — copytree
+ if carra2_final.exists():
+ shutil.rmtree(carra2_final)
+ shutil.copytree(carra2_staging_file, carra2_final)
+ else:
+ # NetCDF or other single-file output
+ shutil.copy2(carra2_staging_file, carra2_final)
+
+ if glacier_groups:
+ # For each S4F group, pre-reproject CARRA2 to that group's CRS at
+ # CARRA2's native ~2.5 km resolution. Uploaded as
+ # ``carra2_.nc`` so ``stage.carra2()`` can fetch a single
+ # small file per glacier instead of streaming the full Zarr and
+ # reprojecting every time.
+ assert complexes is not None # loaded above (need_outlines)
+ for group_name in glacier_groups:
+ row = complexes.loc[complexes["rgi_id"] == group_name]
+ if row.empty:
+ logger.warning("Aggregate complex %s not found in rgi_c.gpkg; skipping CARRA2 prep", group_name)
+ continue
+ group_crs = row["crs"].iloc[0]
+ if not isinstance(group_crs, str) or not group_crs:
+ logger.warning("Aggregate complex %s has no CRS; skipping CARRA2 prep", group_name)
+ continue
+ group_geom = row.geometry.iloc[0]
+ group_out = climate_path / f"carra2_{group_name}.nc"
+ logger.info("Preparing CARRA2 for group %s (%s) -> %s", group_name, group_crs, group_out)
+ prepare_carra2_for_group(
+ carra2_zarr=carra2_final,
+ dst_crs=group_crs,
+ geometry=group_geom,
+ geometry_crs=str(complexes.crs),
+ output_file=group_out,
+ force_overwrite=force_overwrite,
+ )
+
+ return rgi_files
+
+
+def rgi(argv: Sequence[str] | None = None) -> dict[str, Any]:
+ """
+ Prepare RGI glacier input data sets.
+
+ This function is the programmatic entry point. It parses command-line style
+ arguments, creates the target grid, downloads and processes observation data,
+ and prepares climate/ocean forcing files for PISM simulations.
+
+ Parameters
+ ----------
+ argv : sequence of str or None, optional
+ Command-line arguments **excluding** the program name (i.e., like
+ ``sys.argv[1:]``). If ``None`` (default), arguments are taken from the
+ current process' ``sys.argv[1:]``. Passing ``argv=[]`` is recommended
+ when calling from a Jupyter notebook to avoid ipykernel arguments.
+
+ Returns
+ -------
+ dict[str, Any]
+ Mapping returned by :func:`prepare_rgi` (e.g. ``"rgi_complexes"``
+ and ``"rgi_glaciers"`` paths).
"""
parser = ArgumentParser()
@@ -101,6 +396,7 @@ def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
type=int,
default=8,
)
+ add_include_argument(parser, RGI_DATASETS)
parser.add_argument("CONFIG_FILE", nargs=1)
parser.add_argument("OUTPUT_PATH", nargs=1)
args = parser.parse_args(list(argv) if argv is not None else None)
@@ -111,12 +407,9 @@ def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
output_path = Path(args.OUTPUT_PATH[0])
output_path.mkdir(parents=True, exist_ok=True)
- log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
- logging.basicConfig(level=logging.INFO, format=log_format)
- file_handler = logging.FileHandler(output_path / "prepare.log")
- file_handler.setLevel(logging.INFO)
- file_handler.setFormatter(logging.Formatter(log_format))
- logging.getLogger().addHandler(file_handler)
+ setup_logging(output_path / "prepare.log")
+
+ selected = select_datasets(args.include, RGI_DATASETS)
f = Figlet(font="standard")
banner = f.renderText("pism-terra")
@@ -127,35 +420,149 @@ def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
logger.info("-" * 120)
config = toml.loads(Path(config_file).read_text("utf-8"))
- regions = pd.DataFrame.from_dict(config["regions"], orient="index", columns=["name"])
+ regions = pd.DataFrame.from_dict(config["regions"], orient="index")
regions["region"] = regions.index.astype(str).str.zfill(2) + "_" + regions["name"]
+ # Final outputs land under glacier_path; everything intermediate lives in
+ # staging_path so the user can `rm -rf staging` after a clean run without
+ # losing anything that downstream tools need.
glacier_path = output_path / Path("glacier")
glacier_path.mkdir(parents=True, exist_ok=True)
+ staging_path = output_path / Path("staging")
+ staging_path.mkdir(parents=True, exist_ok=True)
+ # --- RGI ---
+ # The RGI outlines are a dependency of the ice-thickness datasets, so the
+ # paths are always resolved; ``prepare_rgi`` only runs when selected,
+ # otherwise the cached gpkg files from a previous run are reused.
rgi_path = glacier_path / Path("rgi")
rgi_path.mkdir(parents=True, exist_ok=True)
+ rgi_staging = staging_path / Path("rgi")
+ rgi_staging.mkdir(parents=True, exist_ok=True)
- rgi_files = prepare_rgi(regions["region"], output_path=rgi_path, force_overwrite=force_overwrite, ntasks=ntasks)
+ rgi_files: dict[str, Any] = {
+ "rgi_complexes": rgi_path / "rgi_c.gpkg",
+ "rgi_glaciers": rgi_path / "rgi_g.gpkg",
+ }
+ if "rgi" in selected:
+ rgi_files = prepare_rgi(
+ regions,
+ glaciers=None,
+ glacier_groups=None,
+ output_path=rgi_path,
+ extract_path=rgi_staging,
+ force_overwrite=force_overwrite,
+ ntasks=ntasks,
+ )
- complexes = gpd.read_file(rgi_files["rgi_complexes"])
- glaciers = gpd.read_file(rgi_files["rgi_glaciers"])
+ # --- Heat flow (Lucazeau 2019) ---
+ if "heatflux_lucazeau" in selected:
+ heatflux_path = glacier_path / Path("heatflux")
+ heatflux_path.mkdir(parents=True, exist_ok=True)
+ heatflux_staging = staging_path / Path("heatflux")
+ heatflux_staging.mkdir(parents=True, exist_ok=True)
+ prepare_heatflux_lucazeau(
+ output_path=heatflux_path,
+ extract_path=heatflux_staging,
+ force_overwrite=force_overwrite,
+ )
- ice_thickness_path = glacier_path / Path("ice_thickness")
- ice_thickness_path.mkdir(parents=True, exist_ok=True)
+ # --- Ice thickness (needs the RGI outlines) ---
+ if {"ice_thickness_frank", "ice_thickness_maffezzoli"} & set(selected):
+ ice_thickness_path = glacier_path / Path("ice_thickness")
+ ice_thickness_path.mkdir(parents=True, exist_ok=True)
+ ice_thickness_staging = staging_path / Path("ice_thickness")
+ ice_thickness_staging.mkdir(parents=True, exist_ok=True)
- maffezzoli_path = ice_thickness_path / Path("maffezzoli")
- maffezzoli_path.mkdir(parents=True, exist_ok=True)
+ complexes = gpd.read_file(rgi_files["rgi_complexes"])
+ glaciers = gpd.read_file(rgi_files["rgi_glaciers"])
- prepare_ice_thickness_maffezzoli(
- regions.index,
- complexes=complexes,
- glaciers=glaciers,
- output_path=maffezzoli_path,
- extract_path=output_path,
- force_overwrite=force_overwrite,
- ntasks=ntasks,
- )
+ if "ice_thickness_frank" in selected:
+ frank_path = ice_thickness_path / Path("frank")
+ frank_path.mkdir(parents=True, exist_ok=True)
+ prepare_ice_thickness_frank(
+ regions.index,
+ complexes=complexes,
+ glaciers=glaciers,
+ output_path=frank_path,
+ extract_path=ice_thickness_staging,
+ rgi_extract_path=rgi_staging,
+ force_overwrite=force_overwrite,
+ ntasks=ntasks,
+ )
+
+ if "ice_thickness_maffezzoli" in selected:
+ maffezzoli_path = ice_thickness_path / Path("maffezzoli")
+ maffezzoli_path.mkdir(parents=True, exist_ok=True)
+ prepare_ice_thickness_maffezzoli(
+ regions.index,
+ complexes=complexes,
+ glaciers=glaciers,
+ output_path=maffezzoli_path,
+ extract_path=ice_thickness_staging,
+ force_overwrite=force_overwrite,
+ ntasks=ntasks,
+ )
+
+ # --- GEBCO ---
+ if "gebco" in selected:
+ # Source NetCDF download lands in staging; only the COG goes to glacier/.
+ gebco_path = glacier_path / Path("gebco")
+ gebco_path.mkdir(parents=True, exist_ok=True)
+ gebco_staging = staging_path / Path("gebco")
+ gebco_staging.mkdir(parents=True, exist_ok=True)
+ gebco_nc = download_gebco(target_dir=gebco_staging)
+ cog_gebco_p = gebco_path / Path("bathymetry.tif")
+
+ # Use xr.open_dataset (CF-aware) so the lat/lon coords become a real
+ # geotransform; rxr.open_rasterio treats netCDF as a generic raster and
+ # loses the georeferencing.
+ ds = xr.open_dataset(gebco_nc, chunks={"lat": 1024, "lon": 1024})
+ da = ds["elevation"].rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=False)
+ if da.rio.crs is None:
+ da = da.rio.write_crs("EPSG:4326")
+ predictor = 3 if np.issubdtype(da.dtype, np.floating) else 2
+ da.rio.to_raster(
+ cog_gebco_p,
+ driver="COG",
+ compress="DEFLATE",
+ predictor=predictor,
+ blocksize=512,
+ bigtiff="YES",
+ overview_resampling="AVERAGE",
+ num_threads="ALL_CPUS",
+ )
+
+ # --- Climate ---
+ if {"snap", "carra2"} & set(selected):
+ climate_path = glacier_path / Path("climate")
+ climate_path.mkdir(parents=True, exist_ok=True)
+
+ if "snap" in selected:
+ # SNAP/CRU-TS40 monthly climatologies (built under staging, copied to
+ # glacier/climate for upload; one file per 30-year window).
+ snap_staging = staging_path / Path("snap")
+ snap_staging.mkdir(parents=True, exist_ok=True)
+ for snap_file in prepare_snap(snap_staging, force_overwrite=force_overwrite):
+ shutil.copy2(snap_file, climate_path / Path(snap_file).name)
+
+ if "carra2" in selected:
+ # Run the download/merge under staging, then move only the merged
+ # product into glacier/climate. Year-by-year CDS intermediates stay
+ # in staging.
+ carra2_staging = staging_path / Path("carra2")
+ carra2_staging.mkdir(parents=True, exist_ok=True)
+
+ carra2_staging_file = prepare_carra2(carra2_staging)
+ carra2_final = climate_path / Path(carra2_staging_file.name)
+ if carra2_staging_file.is_dir():
+ # Zarr store — copytree
+ if carra2_final.exists():
+ shutil.rmtree(carra2_final)
+ shutil.copytree(carra2_staging_file, carra2_final)
+ else:
+ # NetCDF or other single-file output
+ shutil.copy2(carra2_staging_file, carra2_final)
return rgi_files
@@ -174,7 +581,7 @@ def cli(argv: Sequence[str] | None = None) -> int:
int
Exit code (0 for success).
"""
- _ = main(argv=argv)
+ _ = rgi(argv=argv)
return 0
diff --git a/pism_terra/glacier/render_terrain_3d.py b/pism_terra/glacier/render_terrain_3d.py
new file mode 100644
index 0000000..6f23da4
--- /dev/null
+++ b/pism_terra/glacier/render_terrain_3d.py
@@ -0,0 +1,518 @@
+#!/usr/bin/env python
+"""
+Render a 3D terrain from a PISM/netCDF spatial file and drape a field over it.
+
+Builds a 3D surface from a terrain variable (default ``usurf``) and colors it by
+a draped variable (default ``velsurf_mag``), one PNG per time step so the frames
+can be turned into an animation.
+
+Examples
+--------
+ python render_terrain_3d.py path/to/spatial.nc
+ python render_terrain_3d.py spatial.nc --z-exaggeration 4 --cmap turbo --log
+ # then, e.g.:
+ ffmpeg -framerate 15 -i frames/frame_%04d.png -pix_fmt yuv420p out.mp4
+"""
+
+from __future__ import annotations
+
+import argparse
+import platform
+import re
+import subprocess
+from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
+
+import matplotlib
+import numpy as np
+import pyvista as pv
+import xarray as xr
+
+from pism_terra.colormaps import register_colormaps
+
+# Register the project's QGIS colormaps (e.g. "speed") into matplotlib's registry.
+register_colormaps()
+
+
+def detect_screen_size(default: tuple[int, int] = (1600, 1200)) -> tuple[int, int]:
+ """
+ Return the primary display's native pixel resolution.
+
+ On macOS, parses ``system_profiler SPDisplaysDataType``; falls back to
+ ``default`` on any other platform or if detection fails.
+
+ Parameters
+ ----------
+ default : tuple of int, default ``(1600, 1200)``
+ Fallback ``(width, height)`` when the resolution can't be detected.
+
+ Returns
+ -------
+ tuple of int
+ ``(width, height)`` in pixels.
+ """
+ if platform.system() == "Darwin":
+ try:
+ out = subprocess.run(
+ ["system_profiler", "SPDisplaysDataType"],
+ capture_output=True,
+ text=True,
+ timeout=15,
+ check=False,
+ ).stdout
+ m = re.search(r"Resolution:\s*(\d+)\s*x\s*(\d+)", out)
+ if m:
+ return (int(m.group(1)), int(m.group(2)))
+ except (OSError, subprocess.SubprocessError):
+ pass
+ return default
+
+
+def resolve_cmap(name: str) -> matplotlib.colors.Colormap:
+ """
+ Resolve a colormap name to a matplotlib ``Colormap`` object.
+
+ Passing the object (rather than the name) to PyVista bypasses its built-in
+ cmocean/colorcet name handling, so project colormaps registered via
+ :func:`register_colormaps` (e.g. ``"speed"``) are used instead of a
+ same-named cmocean map.
+
+ Parameters
+ ----------
+ name : str
+ Registered colormap name.
+
+ Returns
+ -------
+ matplotlib.colors.Colormap
+ The resolved colormap.
+
+ Raises
+ ------
+ SystemExit
+ If ``name`` is not a registered colormap.
+ """
+ try:
+ return matplotlib.colormaps[name]
+ except KeyError as exc:
+ raise SystemExit(f"Unknown colormap {name!r}. Registered: {', '.join(sorted(matplotlib.colormaps))}") from exc
+
+
+def parse_args() -> argparse.Namespace:
+ """
+ Parse command-line arguments.
+
+ Returns
+ -------
+ argparse.Namespace
+ Parsed arguments.
+ """
+ p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ p.add_argument("infile", help="Input netCDF file.")
+ p.add_argument("-o", "--outdir", default=None, help="Output directory for PNGs (default: _frames).")
+ p.add_argument("--surface-var", default="usurf", help="Variable used for terrain height (default: usurf).")
+ p.add_argument("--z-exaggeration", type=float, default=3.0, help="Vertical exaggeration factor (default: 3).")
+ # Base terrain layer (elevation).
+ p.add_argument("--base-var", default="usurf", help="Variable colored on the base terrain (default: usurf).")
+ p.add_argument("--base-cmap", default="dem_ak", help="Colormap for the base terrain (default: dem_ak).")
+ p.add_argument("--base-clim", type=float, nargs=2, default=None, help="Base color limits MIN MAX (default: auto).")
+ # Overlay layer (velocity on ice).
+ p.add_argument("--overlay-var", default="velsurf_mag", help="Variable overlaid on ice (default: velsurf_mag).")
+ p.add_argument("--overlay-cmap", default="speed", help="Colormap for the overlay (default: speed).")
+ p.add_argument(
+ "--overlay-clim", type=float, nargs=2, default=None, help="Overlay limits MIN MAX (default: robust)."
+ )
+ p.add_argument("--overlay-log", action="store_true", help="Log-scale the overlay color mapping.")
+ p.add_argument(
+ "--overlay-thk",
+ type=float,
+ default=10.0,
+ help="Overlay the velocity only where thk > this (m); default 10.",
+ )
+ p.add_argument("--time-stride", type=int, default=1, help="Render every Nth time step (default: 1).")
+ p.add_argument(
+ "--font-size",
+ type=int,
+ default=None,
+ help="Time-label font size in points (default: auto-scaled to the image height).",
+ )
+ p.add_argument(
+ "--window-size",
+ type=int,
+ nargs=2,
+ default=None,
+ help="PNG size W H (default: detected screen resolution).",
+ )
+ p.add_argument("--azimuth", type=float, default=45.0, help="Camera azimuth from isometric (default: 0).")
+ p.add_argument("--elevation", type=float, default=15.0, help="Camera elevation from isometric (default: 15).")
+ p.add_argument(
+ "--zoom", type=float, default=1.6, help="Camera zoom factor; >1 tightens onto the terrain (default: 1.6)."
+ )
+ p.add_argument(
+ "--aa",
+ choices=["ssaa", "msaa", "fxaa", "none"],
+ default="ssaa",
+ help="Anti-aliasing: ssaa (best/slowest) ... fxaa/none (fastest). Big speed lever.",
+ )
+ p.add_argument(
+ "-j",
+ "--jobs",
+ type=int,
+ default=1,
+ help=(
+ "Parallel rendering processes (default: 1). Speeds up headless software "
+ "rendering (Linux/OSMesa) but is slower on macOS where the GPU serializes."
+ ),
+ )
+ return p.parse_args()
+
+
+def robust_clim(values: np.ndarray, log: bool) -> tuple[float, float]:
+ """
+ Return robust color limits from the 2nd/98th percentiles of finite data.
+
+ Parameters
+ ----------
+ values : numpy.ndarray
+ Data array; non-finite values are ignored.
+ log : bool
+ If True, restrict to positive values and use the 2nd percentile as the
+ lower bound (for a log color scale); otherwise the lower bound is 0.
+
+ Returns
+ -------
+ tuple of float
+ ``(min, max)`` color limits.
+ """
+ finite = values[np.isfinite(values)]
+ if log:
+ finite = finite[finite > 0]
+ if finite.size == 0:
+ return (0.0, 1.0)
+ lo = float(np.percentile(finite, 2)) if log else 0.0
+ hi = float(np.percentile(finite, 98))
+ if not log:
+ lo = min(lo, hi)
+ if hi <= lo:
+ hi = lo + 1.0
+ return (max(lo, 1e-6) if log else lo, hi)
+
+
+def time_label(ds: xr.Dataset, t: int) -> str:
+ """
+ Format a human-readable label for a time step.
+
+ Parameters
+ ----------
+ ds : xarray.Dataset
+ Dataset providing the ``time`` coordinate.
+ t : int
+ Time-step index.
+
+ Returns
+ -------
+ str
+ Formatted date/time (or ``"step "`` if no time coordinate).
+ """
+ if "time" not in ds:
+ return f"step {t}"
+ val = ds["time"].values[t]
+ try: # datetime64 / cftime
+ return str(np.datetime_as_string(val, unit="D"))
+ except (TypeError, ValueError):
+ return str(getattr(val, "isoformat", lambda: val)()) if hasattr(val, "isoformat") else f"{float(val):.1f}"
+
+
+def full_clim(values: np.ndarray) -> tuple[float, float]:
+ """
+ Return the 1st/99th-percentile range of finite data (for elevation).
+
+ Parameters
+ ----------
+ values : numpy.ndarray
+ Data array; non-finite values are ignored.
+
+ Returns
+ -------
+ tuple of float
+ ``(min, max)`` color limits.
+ """
+ finite = values[np.isfinite(values)]
+ if finite.size == 0:
+ return (0.0, 1.0)
+ lo, hi = float(np.percentile(finite, 1)), float(np.percentile(finite, 99))
+ return (lo, hi if hi > lo else lo + 1.0)
+
+
+def bar_args(title: str, position_x: float) -> dict:
+ """
+ Scalar-bar layout for a horizontal bar in the lower half of the frame.
+
+ Parameters
+ ----------
+ title : str
+ Scalar-bar title.
+ position_x : float
+ Left edge of the bar in normalized viewport coordinates (0-1).
+
+ Returns
+ -------
+ dict
+ Keyword arguments for ``Plotter.add_mesh(scalar_bar_args=...)``.
+ """
+ return {
+ "title": title,
+ "color": "black",
+ "n_labels": 5,
+ "vertical": False,
+ "position_x": position_x,
+ "position_y": 0.04,
+ "width": 0.4,
+ "height": 0.05,
+ }
+
+
+def _var_title(ds: xr.Dataset, var: str) -> str:
+ """
+ Build a scalar-bar title ``var [units]`` for a dataset variable.
+
+ Parameters
+ ----------
+ ds : xarray.Dataset
+ Dataset containing ``var``.
+ var : str
+ Variable name.
+
+ Returns
+ -------
+ str
+ ``" []"``, or just ``""`` when no units attribute.
+ """
+ u = ds[var].attrs.get("units", "")
+ return f"{var} [{u}]" if u else var
+
+
+# Per-process state, populated by ``_setup_worker`` (reused across frames).
+_WORKER: dict = {}
+
+
+def _setup_worker(infile: str, cfg: dict) -> None:
+ """
+ Open the dataset and build a reusable off-screen Plotter for this process.
+
+ Run once per worker (the ``ProcessPoolExecutor`` initializer); subsequent
+ :func:`_render_frame` calls reuse the open dataset and Plotter so the GL
+ context and grid coordinates are created only once.
+
+ Parameters
+ ----------
+ infile : str
+ Path to the input netCDF file.
+ cfg : dict
+ Render configuration (variables, colormaps, clims, camera, window
+ size, etc.) shared across all frames.
+ """
+ ds = xr.open_dataset(infile, decode_coords="all")
+ xx, yy = np.meshgrid(ds["x"].values, ds["y"].values)
+ pl = pv.Plotter(off_screen=True, window_size=list(cfg["window_size"]))
+ pl.set_background("white")
+ if cfg["aa"] != "none":
+ try:
+ pl.enable_anti_aliasing(cfg["aa"])
+ except (ValueError, AttributeError):
+ pass
+ _WORKER.update(
+ ds=ds,
+ xx=xx,
+ yy=yy,
+ pl=pl,
+ cfg=cfg,
+ base_cmap=resolve_cmap(cfg["base_cmap"]),
+ overlay_cmap=resolve_cmap(cfg["overlay_cmap"]),
+ )
+
+
+def _render_frame(t: int) -> str:
+ """
+ Render one time step to a PNG using this process's worker state.
+
+ Parameters
+ ----------
+ t : int
+ Time-step index to render.
+
+ Returns
+ -------
+ str
+ Path to the written PNG.
+ """
+ w = _WORKER
+ ds, xx, yy, pl, cfg = w["ds"], w["xx"], w["yy"], w["pl"], w["cfg"]
+
+ z = np.asarray(ds[cfg["surface_var"]].isel(time=t).values, dtype=float)
+ z = np.nan_to_num(z, nan=float(np.nanmin(z)) if np.isfinite(z).any() else 0.0) * cfg["z_exaggeration"]
+ b = np.asarray(ds[cfg["base_var"]].isel(time=t).values, dtype=float)
+ ov = np.asarray(ds[cfg["overlay_var"]].isel(time=t).values, dtype=float)
+ thk = np.asarray(ds["thk"].isel(time=t).values, dtype=float)
+ if cfg["overlay_log"]:
+ ov = np.where(ov > 0, ov, np.nan)
+
+ grid = pv.StructuredGrid(xx, yy, z)
+ grid[cfg["base_var"]] = b.ravel(order="F")
+ grid[cfg["overlay_var"]] = ov.ravel(order="F")
+ grid["thk"] = thk.ravel(order="F")
+
+ # Ice overlay: keep only cells where thk > threshold, lift it slightly.
+ ice = grid.threshold(cfg["overlay_thk"], scalars="thk")
+ if ice.n_points:
+ ice.points[:, 2] += cfg["z_offset"]
+
+ pl.clear()
+ pl.add_mesh(
+ grid,
+ scalars=cfg["base_var"],
+ cmap=w["base_cmap"],
+ clim=cfg["base_clim"],
+ lighting=True,
+ smooth_shading=True,
+ show_scalar_bar=True,
+ scalar_bar_args=bar_args(_var_title(ds, cfg["base_var"]), position_x=0.05),
+ )
+ if ice.n_points:
+ pl.add_mesh(
+ ice,
+ scalars=cfg["overlay_var"],
+ cmap=w["overlay_cmap"],
+ clim=cfg["overlay_clim"],
+ log_scale=cfg["overlay_log"],
+ lighting=True,
+ smooth_shading=True,
+ show_scalar_bar=True,
+ scalar_bar_args=bar_args(_var_title(ds, cfg["overlay_var"]), position_x=0.55),
+ )
+ pl.add_text(
+ time_label(ds, t),
+ position="upper_left",
+ font_size=cfg["font_size"],
+ color="black",
+ shadow=True,
+ )
+ pl.camera_position = cfg["camera"]["position"]
+ # camera_position carries position/focal/up but NOT the view angle that zoom
+ # changes, so reapply it explicitly to preserve the zoom.
+ pl.camera.view_angle = cfg["camera"]["view_angle"]
+
+ out = Path(cfg["outdir"]) / f"frame_{t:04d}.png"
+ pl.screenshot(str(out))
+ return str(out)
+
+
+def _compute_camera(
+ xx: np.ndarray, yy: np.ndarray, z0: np.ndarray, args: argparse.Namespace, window_size: list
+) -> dict:
+ """
+ Compute the fixed camera from the first frame as a picklable, zoom-preserving dict.
+
+ Parameters
+ ----------
+ xx, yy : numpy.ndarray
+ Meshgrid coordinate arrays for the surface (shape ``(ny, nx)``).
+ z0 : numpy.ndarray
+ First-frame (exaggerated) terrain heights, shape ``(ny, nx)``.
+ args : argparse.Namespace
+ Parsed arguments providing ``azimuth``, ``elevation``, and ``zoom``.
+ window_size : list
+ ``[width, height]`` in pixels for the off-screen render.
+
+ Returns
+ -------
+ dict
+ Camera state with keys ``"position"`` (position/focal/up tuples) and
+ ``"view_angle"`` so the zoom survives broadcast to worker processes.
+ """
+ pl = pv.Plotter(off_screen=True, window_size=list(window_size))
+ pl.add_mesh(pv.StructuredGrid(xx, yy, z0))
+ pl.view_isometric()
+ pl.camera.azimuth += args.azimuth
+ pl.camera.elevation += args.elevation
+ pl.camera.zoom(args.zoom) # shrinks the view angle -> zooms in
+ cam = {
+ "position": [tuple(p) for p in pl.camera_position],
+ "view_angle": float(pl.camera.view_angle),
+ }
+ pl.close()
+ return cam
+
+
+def main() -> None:
+ """Render one PNG per time step, in parallel across processes."""
+ args = parse_args()
+ infile = Path(args.infile).expanduser()
+ outdir = Path(args.outdir) if args.outdir else infile.with_name(infile.stem + "_frames")
+ outdir.mkdir(parents=True, exist_ok=True)
+
+ ds = xr.open_dataset(infile, decode_coords="all")
+ for var in (args.surface_var, args.base_var, args.overlay_var):
+ if var not in ds:
+ raise SystemExit(f"Variable {var!r} not in {infile} (have: {sorted(ds.data_vars)}).")
+ if "thk" not in ds:
+ raise SystemExit(f"'thk' not in {infile}; needed to overlay {args.overlay_var} on ice.")
+
+ window_size = list(args.window_size) if args.window_size else list(detect_screen_size())
+ # Scale the time-label font to the image height (~2.5%) so it stays legible
+ # at high resolution and survives H.264/yuv420p compression.
+ font_size = args.font_size if args.font_size else max(14, round(window_size[1] * 0.025))
+
+ xx, yy = np.meshgrid(ds["x"].values, ds["y"].values) # scalars must ravel(order="F")
+ steps = list(range(0, int(ds.sizes.get("time", 1)), args.time_stride))
+
+ # Fixed color limits (defaults match the elevation/velocity scales).
+ base_clim = tuple(args.base_clim) if args.base_clim else (-2000.0, 3500.0)
+ overlay_clim = tuple(args.overlay_clim) if args.overlay_clim else (1.0, 100.0)
+
+ # Small upward offset so the ice overlay wins the depth test over the base.
+ surf0 = np.asarray(ds[args.surface_var].isel(time=steps[0]).values, dtype=float) * args.z_exaggeration
+ z_all = np.asarray(ds[args.surface_var].values, dtype=float) * args.z_exaggeration
+ z_offset = 0.002 * float(np.nanmax(z_all) - np.nanmin(z_all))
+ camera = _compute_camera(xx, yy, np.nan_to_num(surf0, nan=float(np.nanmin(surf0))), args, window_size)
+ ds.close() # each worker opens its own handle
+
+ cfg = {
+ "surface_var": args.surface_var,
+ "base_var": args.base_var,
+ "overlay_var": args.overlay_var,
+ "base_cmap": args.base_cmap,
+ "overlay_cmap": args.overlay_cmap,
+ "base_clim": base_clim,
+ "overlay_clim": overlay_clim,
+ "overlay_log": args.overlay_log,
+ "overlay_thk": args.overlay_thk,
+ "z_exaggeration": args.z_exaggeration,
+ "z_offset": z_offset,
+ "window_size": window_size,
+ "camera": camera,
+ "outdir": str(outdir),
+ "aa": args.aa,
+ "font_size": font_size,
+ }
+
+ total = len(steps)
+ jobs = max(1, min(args.jobs, total))
+ print(f"Rendering {total} frame(s) at {window_size[0]}x{window_size[1]} with {jobs} process(es)...")
+ if jobs == 1:
+ _setup_worker(str(infile), cfg)
+ for i, t in enumerate(steps, 1):
+ print(f"[{i}/{total}] {_render_frame(t)}")
+ _WORKER["pl"].close()
+ else:
+ with ProcessPoolExecutor(max_workers=jobs, initializer=_setup_worker, initargs=(str(infile), cfg)) as ex:
+ for i, out in enumerate(ex.map(_render_frame, steps), 1):
+ print(f"[{i}/{total}] {out}")
+
+ print(f"\nWrote {total} frame(s) to {outdir}")
+ print("Make an animation, e.g.:")
+ print(f" ffmpeg -framerate 15 -i {outdir}/frame_%04d.png -pix_fmt yuv420p {outdir.name}.mp4")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/pism_terra/glacier/rgi.py b/pism_terra/glacier/rgi.py
index a4bd5d9..1372679 100644
--- a/pism_terra/glacier/rgi.py
+++ b/pism_terra/glacier/rgi.py
@@ -28,6 +28,7 @@
import geopandas as gpd
import pandas as pd
+from shapely.geometry import MultiPolygon, Polygon
from tqdm.auto import tqdm
from pism_terra.download import download_archive, extract_archive
@@ -57,7 +58,7 @@ def get_rgi_url(url_template: str, region: str, outline_type: str = "C") -> str:
def prepare_rgi_region(
- region: str,
+ region,
outline_type: str = "C",
url_template: str = "https://daacdata.apps.nsidc.org/pub/DATASETS/nsidc0770_rgi_v7/regional_files/RGI2000-v7.0-C/RGI2000-v7.0-{outline_type}-{region}.zip",
extract_path: Path | str = "rgi_archive",
@@ -73,14 +74,16 @@ def prepare_rgi_region(
Parameters
----------
- region : str or None
- The region code (e.g., "01_alaska", "06_iceland") used to fill in the URL template.
+ region : pandas.Series or Mapping
+ Region descriptor with a ``"region"`` key (e.g., ``"01_alaska"``) used
+ to fill in the URL template. May optionally include a ``"crs"`` entry
+ to override the auto-derived UTM CRS.
outline_type : str, optional
Either C or G (complex or glacier).
url_template : str, optional
URL template containing a `{region}` placeholder. Defaults to the NSIDC RGI v7 template.
extract_path : str or Path, optional
- Path to the directory where the archive will be extracted. Default is "rgi".
+ Path to the directory where the archive will be extracted. Default is "rgi_archive".
area_threshold : float, optional
Minimum glacier area (in square kilometers) for inclusion in the result. Defaults to 1.0.
force_overwrite : bool, default False
@@ -89,9 +92,9 @@ def prepare_rgi_region(
Returns
-------
geopandas.GeoDataFrame
- A GeoDataFrame of glaciers in the region, each with added columns:
- - "crs": EPSG code string based on hemisphere and UTM zone
- - "epsg_code": Integer EPSG code for reprojection
+ A GeoDataFrame of glaciers in the region with an added column:
+ - "crs": EPSG code string based on hemisphere and UTM zone (or the
+ ``crs`` value from the supplied ``region`` if present).
See Also
--------
@@ -104,30 +107,33 @@ def prepare_rgi_region(
"""
extract_path = Path(extract_path)
- url = get_rgi_url(url_template, region, outline_type)
+ region_code = region["region"]
+ url = get_rgi_url(url_template, region_code, outline_type)
archive_dest = extract_path / Path(url.rsplit("/", 1)[-1])
- logger.info("Downloading RGI region %s (%s)", region, outline_type)
+ logger.info("Downloading RGI region %s (%s)", region_code, outline_type)
archive = download_archive(url, dest=archive_dest, force_overwrite=force_overwrite, verbose=False)
- logger.info("Extracting RGI region %s (%s)", region, outline_type)
+ logger.info("Extracting RGI region %s (%s)", region_code, outline_type)
extract_archive(archive, extract_path, force_overwrite=force_overwrite, verbose=False)
- logger.info("Processing RGI region %s (%s)", region, outline_type)
- rgi = gpd.read_file(extract_path / f"RGI2000-v7.0-{outline_type}-{region}.shp")
+ logger.info("Processing RGI region %s (%s)", region_code, outline_type)
+ rgi = gpd.read_file(extract_path / f"RGI2000-v7.0-{outline_type}-{region_code}.shp")
rgi = rgi[rgi["area_km2"] > area_threshold]
- rgi["epsg"] = rgi.apply(
- lambda row: f"""EPSG:{32600 + int(row["utm_zone"]) if row["cenlat"] >= 0 else 32700 + int(row["utm_zone"])}""",
- axis=1,
- )
- rgi["epsg_code"] = rgi.apply(
- lambda row: f"""{32600 + int(row["utm_zone"]) if row["cenlat"] >= 0 else 32700 + int(row["utm_zone"])}""",
- axis=1,
- )
+ crs_value = region.get("crs") if hasattr(region, "get") else None
+ if isinstance(crs_value, str) and crs_value:
+ rgi["crs"] = crs_value
+ else:
+ rgi["crs"] = rgi.apply(
+ lambda row: f"""EPSG:{32600 + int(row["utm_zone"]) if row["cenlat"] >= 0 else 32700 + int(row["utm_zone"])}""",
+ axis=1,
+ )
return rgi
def prepare_rgi(
- regions: list,
+ regions: pd.DataFrame,
output_path: Path,
+ glaciers: pd.DataFrame | list[str] | None = None,
+ glacier_groups: dict[str, pd.DataFrame] | None = None,
extract_path: Path | str = "rgi_archive",
force_overwrite: bool = False,
ntasks: int = 8,
@@ -142,13 +148,33 @@ def prepare_rgi(
Parameters
----------
- regions : list
- Region codes (e.g. ``["01_alaska", "06_iceland"]``).
+ regions : pandas.DataFrame
+ Table of regions with at least a ``"region"`` column whose values are
+ region codes such as ``"01_alaska"`` or ``"06_iceland"``.
output_path : Path
Root directory for output files. A ``rgi/`` subdirectory is created
inside it.
+ glaciers : pandas.DataFrame, list of str, or None, optional
+ Optional whitelist of RGI IDs. May contain glacier IDs (``...-G-...``),
+ complex IDs (``...-C-...``), or both. If a DataFrame is given, the
+ ``rgi_id`` column is used. The output is restricted to:
+
+ - any complex IDs listed,
+ - any glacier IDs listed, and
+ - the parent complexes of any listed glaciers (so glacier outlines
+ always come paired with their containing complex).
+
+ If None (default), no filtering is applied.
+ glacier_groups : dict[str, pandas.DataFrame] or None, optional
+ Optional mapping ``{aggregate_name: dataframe_of_glacier_ids}``. For
+ each entry, an aggregated "complex" row is added to the output
+ complexes GeoPackage whose geometry is the multipolygon union of all
+ parent complexes containing the listed glaciers, and a
+ ``rgi_id_c_aggregate`` column on the output glaciers GeoPackage links
+ each listed glacier back to that aggregate. Useful for study-area
+ rollups (e.g. ``S4F_AK``, ``S4F_CA``).
extract_path : str or Path, optional
- Path to the directory where the archive will be extracted. Default is "rgi".
+ Path to the directory where the archive will be extracted. Default is "rgi_archive".
force_overwrite : bool, default False
If True, re-download the file even if it already exists locally.
ntasks : int, default 8
@@ -178,27 +204,27 @@ def prepare_rgi(
futures = {
executor.submit(
prepare_rgi_region,
- region,
+ row,
outline_type=outline_type,
url_template=url_template,
extract_path=rgi_archive_path,
force_overwrite=force_overwrite,
- ): (region, outline_type)
- for region in regions
+ ): (row["region"], outline_type)
+ for _, row in regions.iterrows()
for outline_type in outline_types
}
pbar = tqdm(cf_as_completed(futures), total=len(futures), desc="Downloading RGI regions")
for future in pbar:
- region, outline_type = futures[future]
+ region_code, outline_type = futures[future]
try:
rgis.append(future.result())
- pbar.set_postfix_str(f"{region} ({outline_type}) ✓")
+ pbar.set_postfix_str(f"{region_code} ({outline_type}) ✓")
except Exception as err:
- failed.append((region, outline_type, err))
- pbar.set_postfix_str(f"{region} ({outline_type}) ✗")
- for region, outline_type, exc in failed:
- logger.error("Failed region: %s, outline_type: %s with error: %s", region, outline_type, exc)
+ failed.append((region_code, outline_type, err))
+ pbar.set_postfix_str(f"{region_code} ({outline_type}) ✗")
+ for region_code, outline_type, exc in failed:
+ logger.error("Failed region: %s, outline_type: %s with error: %s", region_code, outline_type, exc)
logger.info("Concatenating %d RGI dataframes", len(rgis))
rgi = pd.concat(rgis, ignore_index=True)
@@ -246,6 +272,88 @@ def _sjoin_region(region):
if not result.empty:
rgi_g.loc[result.index, "rgi_id_c"] = result.values
+ if glaciers is not None:
+ if isinstance(glaciers, pd.DataFrame):
+ wanted = glaciers["rgi_id"].astype(str).tolist()
+ else:
+ wanted = [str(g) for g in glaciers]
+
+ wanted_g = {i for i in wanted if "-G-" in i}
+ wanted_c = {i for i in wanted if "-C-" in i}
+
+ if wanted_g:
+ parents = rgi_g.loc[rgi_g["rgi_id"].isin(wanted_g), "rgi_id_c"].dropna().unique()
+ wanted_c.update(parents.tolist())
+
+ # Keep ALL glaciers whose parent complex is in the wanted set, so that
+ # downstream per-complex aggregations (e.g. ice-thickness merging) see
+ # every member glacier of each requested complex — not just the ones
+ # the user listed individually.
+ rgi_c = rgi_c[rgi_c["rgi_id"].isin(wanted_c)].copy()
+ rgi_g = rgi_g[rgi_g["rgi_id_c"].isin(wanted_c) | rgi_g["rgi_id"].isin(wanted_g)].copy()
+ logger.info("Filtered to %d complexes and %d glaciers", len(rgi_c), len(rgi_g))
+
+ # For each input glacier-list group, synthesize an aggregated "complex"
+ # whose geometry is the multipolygon union of all parent complexes
+ # containing the listed glaciers. Useful for study-area rollups
+ # (e.g., S4F_AK, S4F_CA, S4F_SV). Each listed glacier gets a back-link
+ # in a new ``rgi_id_c_aggregate`` column so ``glaciers_in_complex`` and
+ # ice-thickness merging pick them up under the aggregate name.
+ if glacier_groups:
+ if "rgi_id_c_aggregate" not in rgi_g.columns:
+ rgi_g["rgi_id_c_aggregate"] = ""
+
+ extra_rows = []
+ for name, df in glacier_groups.items():
+ g_ids = df["rgi_id"].astype(str).tolist()
+ parent_c = rgi_g.loc[rgi_g["rgi_id"].isin(g_ids), "rgi_id_c"].dropna().unique()
+ parents = rgi_c.loc[rgi_c["rgi_id"].isin(parent_c)]
+ if parents.empty:
+ logger.warning("No parent complexes resolved for group %s", name)
+ continue
+
+ # Geometry: union of parent complexes (in the parents' CRS).
+ merged = parents.geometry.union_all()
+ if isinstance(merged, Polygon):
+ merged = MultiPolygon([merged])
+
+ # Carry over the (most common) crs / epsg_code from parents.
+ crs_value = parents["crs"].mode().iloc[0] if "crs" in parents.columns else None
+ epsg_value = parents["epsg_code"].mode().iloc[0] if "epsg_code" in parents.columns else None
+
+ # Recompute area in the inherited projected CRS for the aggregate.
+ area_km2: float | None
+ if crs_value:
+ area_km2 = float(gpd.GeoSeries([merged], crs=rgi_c.crs).to_crs(crs_value).geometry.area.sum()) / 1e6
+ else:
+ area_km2 = float(parents.get("area_km2", pd.Series(dtype=float)).sum()) or None
+
+ extra_rows.append(
+ {
+ "rgi_id": name,
+ "geometry": merged,
+ "crs": crs_value,
+ "epsg_code": epsg_value,
+ "area_km2": area_km2,
+ }
+ )
+
+ # Link each listed glacier to this aggregate. Semicolon-separated
+ # so a glacier can belong to multiple aggregates without row dup.
+ sel = rgi_g["rgi_id"].isin(g_ids)
+ existing = rgi_g.loc[sel, "rgi_id_c_aggregate"].fillna("")
+ updated = existing.where(existing == "", existing + ";") + name
+ rgi_g.loc[sel, "rgi_id_c_aggregate"] = updated
+
+ if extra_rows:
+ extras = gpd.GeoDataFrame(extra_rows, geometry="geometry", crs=rgi_c.crs)
+ rgi_c = gpd.GeoDataFrame(pd.concat([rgi_c, extras], ignore_index=True), geometry="geometry", crs=rgi_c.crs)
+ logger.info(
+ "Added %d aggregated complexes: %s",
+ len(extra_rows),
+ ", ".join(r["rgi_id"] for r in extra_rows),
+ )
+
complex_path = output_path / "rgi_c.gpkg"
logger.info("Saving complexes to %s", complex_path)
rgi_c.to_file(complex_path)
diff --git a/pism_terra/glacier/run.py b/pism_terra/glacier/run.py
index 0328802..f52f988 100644
--- a/pism_terra/glacier/run.py
+++ b/pism_terra/glacier/run.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2025 Andy Aschwanden
+# Copyright (C) 2025-26 Andy Aschwanden
#
# This file is part of pism-terra.
#
@@ -35,42 +35,40 @@
from pyfiglet import Figlet
from pism_terra.aws import local_to_s3
-from pism_terra.config import JobConfig, RunConfig, load_config, load_uq
+from pism_terra.config import JobConfig, load_config, load_uq
from pism_terra.download import file_localizer
from pism_terra.glacier.climate import create_offset_file
from pism_terra.glacier.execute import find_first_and_execute
from pism_terra.glacier.stage import stage_glacier
-from pism_terra.sampling import create_samples
+from pism_terra.sampling import generate_samples
from pism_terra.workflow import (
apply_choice_mapping,
dict2str,
- merge_model,
+ filter_overrides_by_config,
normalize_row,
sort_dict_by_key,
+ validate_pism_options,
)
# one Jinja environment for all renders
_JINJA = Environment(undefined=StrictUndefined, autoescape=False)
-def run_glacier(
+def _render_inverse_run(
rgi_id: str,
config_file: str | Path,
template_file: Path | str,
- outline_file: Path | str,
+ outline_file: Path | str | None,
path: str | Path = "result",
- resolution: None | str = None,
- nodes: None | int = None,
- ntasks: None | int = None,
- queue: None | str = None,
- walltime: None | str = None,
+ config_cli: dict | None = None,
debug: bool = False,
*,
uq: Mapping[str, object] | pd.Series | None = None,
- sample: str | int | None = None,
+ sample: int | None = None,
+ pism_config_cdl: str | Path | None = None,
):
"""
- Configure and generate a PISM job script for a single glacier (ensemble-ready).
+ Configure and generate a PISM inverse job script for a single glacier (ensemble-ready).
Reads a TOML configuration, merges optional ensemble overrides (``uq``),
renders a submission script from a Jinja2 template, and writes both the
@@ -94,19 +92,14 @@ def run_glacier(
path : str or pathlib.Path, optional
Base output directory. A subfolder ``/`` is created with
``output/`` and ``run_scripts/`` subdirectories. Default is ``"result"``.
- resolution : str or None, optional
- Grid resolution (e.g., ``"200m"``). If ``None``, the value from
- ``[grid].resolution`` in the config is used.
- nodes : int or None, optional
- Node count override for the submission template. If ``None``, use config.
- ntasks : int or None, optional
- MPI task count override for the submission template/run options.
- If ``None``, use config.
- queue : str or None, optional
- Batch queue/partition override for the submission template. If ``None``,
- use config.
- walltime : str or None, optional
- Wall time override in ``HH:MM:SS``. If ``None``, use config.
+ config_cli : dict or None, optional
+ CLI-side overrides applied after reading the config. Recognized keys:
+ ``"resolution"`` (e.g. ``"200m"``), ``"nodes"`` (int), ``"ntasks"``
+ (int), ``"tasks"`` (int, MPI tasks per node), ``"queue"`` (str),
+ ``"walltime"`` (``HH:MM:SS``), ``"stress_balance"`` (sub-model name
+ swap, e.g. ``"sia"``), and ``"start"`` / ``"end"`` (``YYYY-MM-DD``
+ time bounds). Any value of ``None`` falls back to the config file.
+ Default is ``None`` (no overrides).
debug : bool, optional
If ``True``, skip rendering the template (leave it empty) but still
append the constructed PISM command line to the output script.
@@ -122,6 +115,9 @@ def run_glacier(
``"sample"``, that value is used. The value changes the filename
stem used for outputs (e.g., ``..._s0042``). If neither is provided,
filenames use a descriptive ``surface/energy/stress_balance`` suffix.
+ pism_config_cdl : str or Path or None, optional
+ Path to a PISM CDL master config file. If provided, all run options
+ are validated against it before generating the command line.
Raises
------
@@ -144,7 +140,7 @@ def run_glacier(
--------
Basic use with config and template:
- >>> run_glacier(
+ >>> run_forward(
... rgi_id="RGI2000-v7.0-C-01-04374",
... config_file="config/init_stampede3.toml",
... template_file="templates/stampede3.j2",
@@ -154,7 +150,7 @@ def run_glacier(
Ensemble member with overrides from a pandas row (e.g., Latin Hypercube):
>>> row = df_samples.loc[17] # contains dotted keys + 'sample'
- >>> run_glacier(
+ >>> run_forward(
... rgi_id="RGI2000-v7.0-C-01-04374",
... config_file="config/init_stampede3.toml",
... template_file="templates/stampede3.j2",
@@ -164,9 +160,19 @@ def run_glacier(
... )
"""
- outline_file = Path(outline_file)
+ outline_file = str(Path(outline_file).resolve()) if (outline_file is not None) else "none"
+ # Derive the complex (-C) and glacier (-G) outline paths (co-located in the
+ # staging dir) so postprocessing can clip to either. See stage.py.
+ if outline_file != "none":
+ _outline_dir = Path(outline_file).parent
+ outline_c_file = str((_outline_dir / f"rgi_{rgi_id}-C.gpkg").resolve())
+ outline_g_file = str((_outline_dir / f"rgi_{rgi_id}-G.gpkg").resolve())
+ else:
+ outline_c_file = outline_g_file = "none"
cfg = load_config(config_file)
+ config_cli = config_cli or {}
+ resolution = config_cli.get("resolution")
if resolution:
resolution = re.sub(r"\s+", "", resolution)
@@ -189,11 +195,19 @@ def run_glacier(
spatial_path.mkdir(parents=True, exist_ok=True)
state_path = output_path / Path("state")
state_path.mkdir(parents=True, exist_ok=True)
+ inv_path = output_path / Path("inverse")
+ inv_path.mkdir(parents=True, exist_ok=True)
+
+ # CLI override for the stress-balance model. Apply before assembling the
+ # forward (``run``) and inverse (``inv``) option dicts so both pick up the
+ # swapped model — otherwise the pismi call keeps the config's model.
+ stress_balance_cli = config_cli.get("stress_balance")
+ if stress_balance_cli is not None:
+ cfg.stress_balance.model = stress_balance_cli
run = {}
for section in (
"geometry",
- "ocean",
"calving",
"iceflow",
"reporting",
@@ -203,22 +217,65 @@ def run_glacier(
run.update(getattr(cfg, section))
run.update(cfg.stress_balance.selected())
run.update(cfg.atmosphere.selected())
+ run.update(cfg.ocean.selected())
run.update(cfg.surface.selected())
run.update(cfg.energy.selected())
run.update(cfg.grid.as_params())
run.update(cfg.run_info.as_params())
run.update(cfg.time.as_params())
+ # Forward solver knobs ([solver.forward]) drive the forward pism call.
+ run.update(cfg.solver.get("forward", {}))
+
+ inv = {}
+ inv.update(getattr(cfg, "iceflow"))
+ inv.update(getattr(cfg, "inverse"))
+ # Inverse solver knobs ([solver.inverse]) drive the pismi call.
+ inv.update(cfg.solver.get("inverse", {}))
+
+ # cfg.stress_balance.selected() carries everything the forward run needs
+ # (model options + PETSc solver knobs like bp_* / inv_adj_*). The pismi
+ # call only needs the ``stress_balance.*`` dotted options; the solver
+ # flags are picked up by the prior pism call (and inherited from the
+ # state file). Filter so inv_str stays minimal.
+ inv.update({k: v for k, v in cfg.stress_balance.selected().items() if k.startswith("stress_balance.")})
+
+ # The energy block defines the ice rheology (``energy.model`` and the
+ # ``flow_law``/ice-softness settings), which the Blatter forward and adjoint
+ # solves in pismi depend on. It is pure physics (no PETSc solver knobs), so
+ # forward it whole; without it the inverse run silently uses PISM's default
+ # rheology instead of the configured one.
+ inv.update(cfg.energy.selected())
template_file = Path(template_file)
env = Environment(loader=FileSystemLoader(template_file.parent))
template = env.get_template(template_file.name)
+ # CLI overrides for time bounds. ``cfg.time`` is a TimeConfig pydantic
+ # model with field names ``time_start`` / ``time_end`` (aliased to the
+ # dotted ``"time.start"`` / ``"time.end"``), so we set attributes, not
+ # items. We drop the prior value from ``run`` first and re-apply via
+ # ``as_params()`` so the dotted alias replaces cleanly.
+ _start = config_cli.get("start")
+ _end = config_cli.get("end")
+ if _start is not None:
+ run.pop("time.start", None)
+ cfg.time.time_start = _start
+ run.update(cfg.time.as_params())
+
+ if _end is not None:
+ run.pop("time.end", None)
+ cfg.time.time_end = _end
+ run.update(cfg.time.as_params())
+
start = cfg.model_dump(by_alias=True)["time"]["time.start"]
end = cfg.model_dump(by_alias=True)["time"]["time.end"]
if resolution is None:
resolution = cfg.model_dump(by_alias=True)["grid"]["resolution"]
+ # ``cfg.stress_balance.model`` was already swapped (if requested) before the
+ # option dicts were built, so the generated command(s) reflect the CLI choice.
stress_balance = cfg.model_dump(by_alias=True)["stress_balance"]["model"]
+
energy = cfg.model_dump(by_alias=True)["energy"]["model"]
surface = cfg.model_dump(by_alias=True)["surface"]["model"]
@@ -235,10 +292,20 @@ def run_glacier(
except Exception:
pass
- # Remove 'sample' from flag overrides
- overrides = {k: v for k, v in uq_clean.items() if k != "sample"}
- # Apply to runtime dict (these should be dotted PISM flags)
- run.update(overrides)
+ # Remove 'sample' from flag overrides; drop any key not in either the
+ # ``run`` or ``inv`` dicts (e.g., surface.debm_simple.std_dev.file when
+ # surface.model == "pdd"). ``inverse.*`` keys live in ``inv`` only, so
+ # filtering against ``run.keys()`` alone would silently drop them — that
+ # was the bug that kept ``inverse.file`` stuck at "none".
+ all_overrides = {k: v for k, v in uq_clean.items() if k != "sample"}
+ run_overrides, _ = filter_overrides_by_config(all_overrides, run.keys())
+ inv_overrides, _ = filter_overrides_by_config(all_overrides, inv.keys())
+ skipped = [k for k in all_overrides if k not in run and k not in inv]
+ if skipped:
+ print(f"Skipping uq overrides not in config: {skipped}")
+ # Apply to both runtime dicts (these should be dotted PISM flags)
+ run.update(run_overrides)
+ inv.update(inv_overrides)
scalar_file = scalar_path / Path(f"scalar_g{resolution}_{rgi_id}_{name_options}_{start}_{end}.nc")
spatial_file = spatial_path / Path(f"spatial_g{resolution}_{rgi_id}_{name_options}_{start}_{end}.nc")
@@ -251,34 +318,41 @@ def run_glacier(
}
)
+ if pism_config_cdl is not None:
+ validate_pism_options(run, pism_config_cdl)
+
run_str = dict2str(sort_dict_by_key(run))
- run_opts = RunConfig(**cfg.run.model_dump())
+ inv_file = inv_path / Path(f"inv_g{resolution}_{rgi_id}_{name_options}_{start}_{end}.nc")
+ # Feed the forward run's state file into pismi as its input.
+ inv.update({"input.file": state_file.resolve()})
+ # inverse output file
+ inv.update({"o": inv_file.resolve()})
+ inv_str = dict2str(sort_dict_by_key(inv))
+
job_opts = JobConfig(**cfg.job.model_dump())
params = {
- **run_opts.model_dump(exclude_none=True, by_alias=True),
**job_opts.model_dump(exclude_none=True, by_alias=True),
}
- # run_opts comes from your config; ntasks comes from CLI (or None)
- active_run_opts = merge_model(run_opts, ntasks=ntasks)
-
- # Use this ONE source to update params and to compute mpi_str
- run_params = active_run_opts.as_params()
- params.update(run_params)
- mpi_str = run_params["mpi"] # guaranteed consistent with ntasks override
-
job_kwargs = {
k: v
- for k, v in {"queue": queue, "walltime": walltime, "nodes": nodes, "output_path": log_path.resolve()}.items()
+ for k, v in {
+ "nodes": config_cli.get("nodes"),
+ "ntasks": config_cli.get("ntasks"),
+ "queue": config_cli.get("queue"),
+ "output_path": log_path.resolve(),
+ "tasks": config_cli.get("tasks"),
+ "walltime": config_cli.get("walltime"),
+ }.items()
if v is not None
}
if job_kwargs:
params.update(JobConfig(**job_kwargs).as_params())
run_toml = {
- "rgi": {"rgi_id": rgi_id, "outline": str(outline_file.resolve())},
+ "rgi": {"rgi_id": rgi_id, "outline_c": outline_c_file, "outline_g": outline_g_file},
"output": {
"spatial": str(spatial_file.resolve()),
"scalar.file": scalar_file.resolve(),
@@ -293,10 +367,11 @@ def run_glacier(
with open(post_file, "w", encoding="utf-8") as toml_file:
toml.dump(run_toml, toml_file)
- prefix = f"{mpi_str} {cfg.run.executable} "
- postfix = f"pism-glacier-postprocess {post_file}"
+ params.update({"run_str": run_str})
+ params.update({"inv_str": inv_str})
+ params.update({"post_script": "pism-glacier-postprocess"})
+ params.update({"post_file": post_file})
rendered_script = "" if debug else template.render(params)
- rendered_script += f"\n\n{prefix}{run_str}\n\n{postfix}"
run_script_path = glacier_path / Path("run_scripts")
run_script_path.mkdir(parents=True, exist_ok=True)
@@ -310,166 +385,349 @@ def run_glacier(
print(f"Postprocessing script written to {post_file.resolve()}\n")
-def run_single():
+def _render_forward_run(
+ rgi_id: str,
+ config_file: str | Path,
+ template_file: Path | str,
+ outline_file: Path | str | None,
+ path: str | Path = "result",
+ config_cli: dict | None = None,
+ debug: bool = False,
+ *,
+ uq: Mapping[str, object] | pd.Series | None = None,
+ sample: int | None = None,
+ pism_config_cdl: str | Path | None = None,
+):
"""
- Run single glacier.
+ Configure and generate a PISM job script for a single glacier (ensemble-ready).
+
+ Reads a TOML configuration, merges optional ensemble overrides (``uq``),
+ renders a submission script from a Jinja2 template, and writes both the
+ script and a companion TOML describing the resolved run parameters.
+ Also emits a command-line string of PISM flags derived from the config and
+ overrides.
+
+ Parameters
+ ----------
+ rgi_id : str
+ Glacier identifier (e.g., ``"RGI2000-v7.0-C-01-04374"``). Used to build
+ output directory and filenames.
+ config_file : str or pathlib.Path
+ Path to the PISM configuration TOML (contains ``run``, ``grid``,
+ ``time``, ``surface``, ``energy``, ``stress_balance``, etc.).
+ template_file : str or pathlib.Path
+ Path to a Jinja2 submission template (e.g., SLURM/LSF script). The
+ context is populated from validated ``RunConfig`` and ``JobConfig``.
+ outline_file : str or pathlib.Path
+ Path to a geopandas file with the glacier outline.
+ path : str or pathlib.Path, optional
+ Base output directory. A subfolder ``/`` is created with
+ ``output/`` and ``run_scripts/`` subdirectories. Default is ``"result"``.
+ config_cli : dict or None, optional
+ CLI-side overrides applied after reading the config. Recognized keys:
+ ``"resolution"`` (e.g. ``"200m"``), ``"nodes"`` (int), ``"ntasks"``
+ (int), ``"tasks"`` (int, MPI tasks per node), ``"queue"`` (str),
+ ``"walltime"`` (``HH:MM:SS``), ``"stress_balance"`` (sub-model name
+ swap, e.g. ``"sia"``), and ``"start"`` / ``"end"`` (``YYYY-MM-DD``
+ time bounds). Any value of ``None`` falls back to the config file.
+ Default is ``None`` (no overrides).
+ debug : bool, optional
+ If ``True``, skip rendering the template (leave it empty) but still
+ append the constructed PISM command line to the output script.
+ Default is ``False``.
+ uq : Mapping[str, object] or pandas.Series or None, optional
+ Ensemble overrides. Keys are **dotted PISM flags** (e.g.,
+ ``"surface.pdd.factor_ice"``, ``"input.file"``). Values are inserted into
+ the run dictionary and thus into the generated command line. If ``uq``
+ contains a key ``"sample"``, it is used (when ``sample`` is not provided)
+ to suffix output filenames and scripts.
+ sample : int or None, optional
+ Ensemble member identifier. If not provided, and ``uq`` has
+ ``"sample"``, that value is used. The value changes the filename
+ stem used for outputs (e.g., ``..._s0042``). If neither is provided,
+ filenames use a descriptive ``surface/energy/stress_balance`` suffix.
+ pism_config_cdl : str or Path or None, optional
+ Path to a PISM CDL master config file. If provided, all run options
+ are validated against it before generating the command line.
+
+ Raises
+ ------
+ ValueError
+ If configuration validation fails upstream (e.g., via Pydantic models),
+ or if provided overrides are of incompatible types.
+
+ Notes
+ -----
+ - The Jinja2 context is populated from validated ``RunConfig`` and
+ ``JobConfig`` (config values) plus any CLI overrides provided here
+ for ``ntasks``, ``nodes``, ``queue``, ``walltime``.
+ - ``uq`` overrides are merged **after** reading the config; they can set or
+ replace any dotted PISM flag (e.g., swapping input or forcing files).
+ - The function attempts to open NetCDF inputs referenced by keys ending
+ with ``.file`` (excluding ``output.*``) using ``xarray.open_dataset`` and
+ prints a ✓/✗ check; it does not stop the run on failure.
+
+ Examples
+ --------
+ Basic use with config and template:
+
+ >>> run_forward(
+ ... rgi_id="RGI2000-v7.0-C-01-04374",
+ ... config_file="config/init_stampede3.toml",
+ ... template_file="templates/stampede3.j2",
+ ... path="result",
+ ... )
+
+ Ensemble member with overrides from a pandas row (e.g., Latin Hypercube):
+
+ >>> row = df_samples.loc[17] # contains dotted keys + 'sample'
+ >>> run_forward(
+ ... rgi_id="RGI2000-v7.0-C-01-04374",
+ ... config_file="config/init_stampede3.toml",
+ ... template_file="templates/stampede3.j2",
+ ... uq=row, # dotted PISM flags to override
+ ... sample=None, # will be inferred from row['sample'] if present
+ ... ntasks=112, # optional template/run override
+ ... )
"""
- # set up the option parser
- parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
- parser.description = "Stage RGI Glacier."
- parser.add_argument("--bucket", help="AWS S3 Bucket to upload output files to")
- parser.add_argument(
- "--bucket-prefix",
- help="AWS prefix (location in bucket) to add to product files",
- default="",
- )
- parser.add_argument(
- "--output-path",
- help="Base path to save all files to. Files will be saved in `f'{out_path}/{RGI_ID}/output/'`.",
- type=str,
- default="data",
- )
- parser.add_argument(
- "--force-overwrite",
- help="Force downloading all files.",
- action="store_true",
- default=False,
- )
- parser.add_argument(
- "--queue",
- help="Overrides queue in config file.",
- type=str,
- default=None,
- )
- parser.add_argument(
- "--ntasks",
- help="Overrides ntasks in config file.",
- type=int,
- default=None,
- )
- parser.add_argument(
- "--nodes",
- help="Overrides nodes in config file.",
- type=int,
- default=None,
- )
- parser.add_argument(
- "--walltime",
- help="Overrides walltime in config file.",
- type=str,
- default=None,
- )
- parser.add_argument(
- "--resolution",
- help="Override horizontal grid resolution.",
- type=str,
- default=None,
- )
- parser.add_argument(
- "--execute",
- help="Execute the pism run script immediately. Ignored if `--debug` is provided.",
- action="store_true",
- )
- parser.add_argument(
- "--debug",
- help="Debug or testing mode, do not write template, just the run command.",
- action="store_true",
- default=False,
- )
- parser.add_argument(
- "RGI_ID",
- help="RGI ID.",
- nargs="?",
- )
- parser.add_argument(
- "CONFIG_FILE",
- help="CONFIG TOML.",
- nargs="?",
- )
- parser.add_argument(
- "TEMPLATE_FILE",
- help="TEMPLATE J2.",
- nargs="?",
- )
+ outline_file = str(Path(outline_file).resolve()) if (outline_file is not None) else "none"
+ # Derive the complex (-C) and glacier (-G) outline paths (co-located in the
+ # staging dir) so postprocessing can clip to either. See stage.py.
+ if outline_file != "none":
+ _outline_dir = Path(outline_file).parent
+ outline_c_file = str((_outline_dir / f"rgi_{rgi_id}-C.gpkg").resolve())
+ outline_g_file = str((_outline_dir / f"rgi_{rgi_id}-G.gpkg").resolve())
+ else:
+ outline_c_file = outline_g_file = "none"
+ cfg = load_config(config_file)
- options = parser.parse_args()
- force_overwrite = options.force_overwrite
+ config_cli = config_cli or {}
+ resolution = config_cli.get("resolution")
+ if resolution:
+ resolution = re.sub(r"\s+", "", resolution)
- path = Path(options.output_path)
- rgi_id = options.RGI_ID
- glacier_path = path / rgi_id
+ # update GridConfig and force dx/dy to be derived from the new resolution
+ cfg.grid.resolution = resolution
+ cfg.grid.dx = None
+ cfg.grid.dy = None
- input_path = glacier_path / "input"
- input_path.mkdir(parents=True, exist_ok=True)
- output_path = glacier_path / "output"
+ path = Path(path)
+ path.mkdir(parents=True, exist_ok=True)
+ glacier_path = path / Path(rgi_id)
+ glacier_path.mkdir(parents=True, exist_ok=True)
+ log_path = glacier_path / Path("logs")
+ log_path.mkdir(parents=True, exist_ok=True)
+ output_path = glacier_path / Path("output")
output_path.mkdir(parents=True, exist_ok=True)
+ scalar_path = output_path / Path("scalar")
+ scalar_path.mkdir(parents=True, exist_ok=True)
+ spatial_path = output_path / Path("spatial")
+ spatial_path.mkdir(parents=True, exist_ok=True)
+ state_path = output_path / Path("state")
+ state_path.mkdir(parents=True, exist_ok=True)
- config_file = file_localizer(options.CONFIG_FILE, path / "config")
- template_file = file_localizer(options.TEMPLATE_FILE, path / "templates")
- resolution = options.resolution
+ # CLI override for the stress-balance model. Apply before assembling the
+ # ``run`` option dict so it picks up the swapped model.
+ stress_balance_cli = config_cli.get("stress_balance")
+ if stress_balance_cli is not None:
+ cfg.stress_balance.model = stress_balance_cli
- debug = options.debug
- queue = options.queue
- ntasks = options.ntasks
- nodes = options.nodes
- walltime = options.walltime
+ run = {}
+ for section in (
+ "geometry",
+ "calving",
+ "iceflow",
+ "reporting",
+ "input",
+ "inverse",
+ "time_stepping",
+ ):
+ run.update(getattr(cfg, section))
+ run.update(cfg.stress_balance.selected())
+ run.update(cfg.atmosphere.selected())
+ run.update(cfg.ocean.selected())
+ run.update(cfg.surface.selected())
+ run.update(cfg.energy.selected())
+ run.update(cfg.grid.as_params())
+ run.update(cfg.run_info.as_params())
+ run.update(cfg.time.as_params())
+ # Forward solver knobs ([solver.forward]) drive the forward pism call.
+ run.update(cfg.solver.get("forward", {}))
- cfg = load_config(config_file)
- campaign_config = cfg.campaign.as_params()
- df = stage_glacier(campaign_config, rgi_id, path=input_path, force_overwrite=force_overwrite)
-
- default = {
- "input.file": df["boot_file"].iloc[0],
- "grid.file": df["grid_file"].iloc[0],
- "surface.force_to_thickness.file": df["boot_file"].iloc[0],
- "atmosphere.delta_T.file": df["scalar_offset_file"].iloc[0],
- "atmosphere.elevation_change.file": df["climate_file"].iloc[0],
- "atmosphere.fract_P.file": df["scalar_offset_file"].iloc[0],
- "atmosphere.given.file": df["climate_file"].iloc[0],
+ template_file = Path(template_file)
+ env = Environment(loader=FileSystemLoader(template_file.parent))
+ template = env.get_template(template_file.name)
+
+ # CLI overrides for time bounds. ``cfg.time`` is a TimeConfig pydantic
+ # model with field names ``time_start`` / ``time_end`` (aliased to the
+ # dotted ``"time.start"`` / ``"time.end"``), so we set attributes, not
+ # items. We drop the prior value from ``run`` first and re-apply via
+ # ``as_params()`` so the dotted alias replaces cleanly.
+ _start = config_cli.get("start")
+ _end = config_cli.get("end")
+ if _start is not None:
+ run.pop("time.start", None)
+ cfg.time.time_start = _start
+ run.update(cfg.time.as_params())
+
+ if _end is not None:
+ run.pop("time.end", None)
+ cfg.time.time_end = _end
+ run.update(cfg.time.as_params())
+
+ start = cfg.model_dump(by_alias=True)["time"]["time.start"]
+ end = cfg.model_dump(by_alias=True)["time"]["time.end"]
+
+ if resolution is None:
+ resolution = cfg.model_dump(by_alias=True)["grid"]["resolution"]
+ # ``cfg.stress_balance.model`` was already swapped (if requested) before the
+ # option dicts were built, so the generated command(s) reflect the CLI choice.
+ stress_balance = cfg.model_dump(by_alias=True)["stress_balance"]["model"]
+
+ energy = cfg.model_dump(by_alias=True)["energy"]["model"]
+ surface = cfg.model_dump(by_alias=True)["surface"]["model"]
+
+ if sample is None:
+ name_options = f"surface_{surface}_energy_{energy}_stress_balance_{stress_balance}"
+ else:
+ name_options = f"id_{sample}"
+
+ uq_clean = normalize_row(uq) if uq is not None else {}
+ # Prefer explicit `sample` arg; else default from uq['sample']
+ if sample is None and "sample" in uq_clean:
+ try:
+ sample = int(uq_clean["sample"])
+ except Exception:
+ pass
+
+ # Remove 'sample' from flag overrides; drop any key not in the config-derived
+ # run dict (e.g., surface.debm_simple.std_dev.file when surface.model == "pdd").
+ overrides = {k: v for k, v in uq_clean.items() if k != "sample"}
+ overrides, skipped = filter_overrides_by_config(overrides, run.keys())
+ if skipped:
+ print(f"Skipping uq overrides not in config: {skipped}")
+ # Apply to runtime dict (these should be dotted PISM flags)
+ run.update(overrides)
+
+ scalar_file = scalar_path / Path(f"scalar_g{resolution}_{rgi_id}_{name_options}_{start}_{end}.nc")
+ spatial_file = spatial_path / Path(f"spatial_g{resolution}_{rgi_id}_{name_options}_{start}_{end}.nc")
+ state_file = state_path / Path(f"state_g{resolution}_{rgi_id}_{name_options}_{start}_{end}.nc")
+ run.update(
+ {
+ "output.file": state_file.resolve(),
+ "output.scalar.file": scalar_file.resolve(),
+ "output.spatial.file": spatial_file.resolve(),
+ }
+ )
+
+ if pism_config_cdl is not None:
+ validate_pism_options(run, pism_config_cdl)
+
+ run_str = dict2str(sort_dict_by_key(run))
+
+ job_opts = JobConfig(**cfg.job.model_dump())
+
+ params = {
+ **job_opts.model_dump(exclude_none=True, by_alias=True),
}
- outline_file = df["outline"].iloc[0]
- f = Figlet(font="standard")
- banner = f.renderText("pism-terra")
- print("=" * 80)
- print(banner)
- print("=" * 80)
- print(f"Generate Run for Glacier {rgi_id}")
- print("-" * 80)
- for idx, row in df.iterrows():
- run_glacier(
- rgi_id,
- config_file,
- template_file,
- outline_file,
- path=path,
- resolution=resolution,
- nodes=nodes,
- ntasks=ntasks,
- queue=queue,
- walltime=walltime,
- debug=debug,
- uq=default,
- sample=int(row["sample"]) if "sample" in row else idx,
- )
+ job_kwargs = {
+ k: v
+ for k, v in {
+ "nodes": config_cli.get("nodes"),
+ "ntasks": config_cli.get("ntasks"),
+ "queue": config_cli.get("queue"),
+ "output_path": log_path.resolve(),
+ "tasks": config_cli.get("tasks"),
+ "walltime": config_cli.get("walltime"),
+ }.items()
+ if v is not None
+ }
+ if job_kwargs:
+ params.update(JobConfig(**job_kwargs).as_params())
- if options.execute and not options.debug:
- find_first_and_execute(path / rgi_id)
+ run_toml = {
+ "rgi": {"rgi_id": rgi_id, "outline_c": outline_c_file, "outline_g": outline_g_file},
+ "output": {
+ "spatial": str(spatial_file.resolve()),
+ "scalar.file": scalar_file.resolve(),
+ "state": str(state_file.resolve()),
+ },
+ "config": run,
+ }
+ post_path = output_path / Path("post_processing")
+ post_path.mkdir(parents=True, exist_ok=True)
- if options.bucket:
- prefix = f"{options.bucket_prefix}/{rgi_id}" if options.bucket_prefix else rgi_id
- local_to_s3(glacier_path, bucket=options.bucket, prefix=prefix)
+ post_file = post_path / Path(f"g{resolution}_{rgi_id}_{name_options}_{start}_{end}.toml")
+ with open(post_file, "w", encoding="utf-8") as toml_file:
+ toml.dump(run_toml, toml_file)
+ params.update({"run_str": run_str})
+ params.update({"post_script": "pism-glacier-postprocess"})
+ params.update({"post_file": post_file})
+ rendered_script = "" if debug else template.render(params)
-def run_ensemble():
+ run_script_path = glacier_path / Path("run_scripts")
+ run_script_path.mkdir(parents=True, exist_ok=True)
+
+ run_script = run_script_path / Path(f"submit_g{resolution}_{rgi_id}_{name_options}_{start}_{end}.sh")
+
+ # Save or print the output
+ run_script.write_text(rendered_script)
+
+ print(f"\nSLURM script written to {run_script.resolve()}\n")
+ print(f"Postprocessing script written to {post_file.resolve()}\n")
+
+
+def _nullable_string(argument_string: str) -> str | None:
+ """
+ Handle null/None CLI parameters from HyP3.
+
+ There's no way in AWS batch to selectively include parameters, so HyP3 needs
+ special handling for optional (nullable) API parameters. In the docker run command,
+ null arguments will appear as the string `None`. This argparse type ensures all `None`
+ strings are accurately represented as `None` objects in Python.
+
+ Parameters
+ ----------
+ argument_string : str
+ Argument string to parse.
+
+ Returns
+ -------
+ str | None
+ The parsed argument string.
"""
- Run single glacier ensemble.
+ if argument_string.strip().lower() == "none":
+ return None
+
+ return argument_string
+
+
+def _build_cli_parser(description: str, *, supports_execute: bool) -> ArgumentParser:
"""
+ Build the argparse parser shared by ``run_forward`` and ``run_inverse``.
+
+ ``UQ_FILE`` is exposed as an *optional* positional: omit it to render one
+ job script (single mode), supply it to render an ensemble.
- # set up the option parser
+ Parameters
+ ----------
+ description : str
+ Parser description shown in ``--help``.
+ supports_execute : bool
+ Whether to add the ``--execute`` flag. Forward runs accept it (the
+ first generated script is launched in-process); inverse runs may
+ also accept it.
+
+ Returns
+ -------
+ argparse.ArgumentParser
+ Configured parser.
+ """
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
- parser.description = "Stage RGI Glacier Ensemble."
+ parser.description = description
parser.add_argument("--bucket", help="AWS S3 Bucket to upload output files to")
parser.add_argument(
"--bucket-prefix",
@@ -480,7 +738,7 @@ def run_ensemble():
"--output-path",
help="Base path to save all files to. Files will be saved in `f'{out_path}/{RGI_ID}/output/'`.",
type=str,
- default="data",
+ default=".",
)
parser.add_argument(
"--force-overwrite",
@@ -488,69 +746,144 @@ def run_ensemble():
action="store_true",
default=False,
)
+ parser.add_argument("--queue", type=str, default=None, help="Overrides queue in config file.")
+ parser.add_argument("--ntasks", type=int, default=None, help="Numbers of cores.")
+ parser.add_argument("--tasks", type=int, default=None, help="Cores per node.")
+ parser.add_argument("--nodes", type=int, default=None, help="Overrides nodes in config file.")
+ parser.add_argument("--walltime", type=str, default=None, help="Overrides walltime in config file.")
parser.add_argument(
- "--queue",
- help="Overrides queue in config file.",
- type=str,
- default=None,
- )
- parser.add_argument(
- "--ntasks",
- help="Overrides ntasks in config file.",
- type=int,
- default=None,
- )
- parser.add_argument(
- "--nodes",
- help="Overrides nodes in config file.",
- type=int,
- default=None,
- )
- parser.add_argument(
- "--walltime",
- help="Overrides walltime in config file.",
- type=str,
- default=None,
+ "--resolution", type=_nullable_string, default=None, help="Override horizontal grid resolution."
)
parser.add_argument(
- "--resolution",
- help="Override horizontal grid resolution.",
- type=str,
+ "--stress-balance",
+ type=_nullable_string,
default=None,
+ help="Override the [stress_balance].model selection (e.g. 'sia', 'blatter').",
)
+ parser.add_argument("--start", type=_nullable_string, default=None, help="Override the time.start selection.")
+ parser.add_argument("--end", type=_nullable_string, default=None, help="Override the time.end selection.")
parser.add_argument(
"--posterior-file",
- help="CSV file posterior parameter distributions to sample from. Default=None.",
- type=str,
+ type=_nullable_string,
default=None,
+ help="CSV file of posterior parameter distributions to sample from (ensemble mode only).",
)
+ if supports_execute:
+ parser.add_argument(
+ "--execute",
+ action="store_true",
+ help="Execute the first generated run script in-process. Ignored in ensemble mode or with --debug.",
+ )
parser.add_argument(
"--debug",
- help="Debug or testing mode, do not write template, just the run command.",
action="store_true",
default=False,
+ help="Debug or testing mode, do not write template, just the run command.",
)
parser.add_argument(
- "RGI_ID",
- help="RGI ID.",
- nargs="?",
- )
- parser.add_argument(
- "CONFIG_FILE",
- help="CONFIG TOML.",
- nargs="?",
- )
- parser.add_argument(
- "TEMPLATE_FILE",
- help="TEMPLATE J2.",
- nargs="?",
+ "--pism-config-cdl",
+ type=_nullable_string,
+ default=None,
+ help="Path to PISM CDL config file for option validation.",
)
+ parser.add_argument("RGI_ID", help="RGI ID.")
+ parser.add_argument("CONFIG_FILE", help="CONFIG TOML.")
+ parser.add_argument("TEMPLATE_FILE", help="TEMPLATE J2.")
parser.add_argument(
"UQ_FILE",
- help="UQ TOML.",
nargs="?",
+ default=None,
+ type=_nullable_string,
+ help="UQ TOML (optional). Supply to render an ensemble; omit for a single-glacier run.",
)
+ return parser
+
+
+def _build_ensemble_df(
+ df: pd.DataFrame,
+ uq_file: Path,
+ output_path: Path,
+ posterior_file: str | Path | None,
+ seed: int = 42,
+) -> pd.DataFrame:
+ """
+ Build the per-member DataFrame for an ensemble run.
+
+ Samples the UQ specification, optionally folds in a posterior CSV, then
+ cross-joins with the staged glacier DataFrame ``df`` and assigns a
+ composite ``sample`` ID per row.
+
+ Parameters
+ ----------
+ df : pandas.DataFrame
+ Output of :func:`pism_terra.glacier.stage.stage_glacier` (one row
+ per staged ``boot``/``grid``/``climate`` tuple).
+ uq_file : Path
+ Path to the UQ TOML.
+ output_path : Path
+ Directory under which the realised sample CSV is persisted.
+ posterior_file : str or Path or None
+ Optional CSV of posterior parameter draws to override / extend the
+ UQ samples with.
+ seed : int, default 42
+ Seed for sampling (and posterior row choice).
+
+ Returns
+ -------
+ pandas.DataFrame
+ Per-ensemble-member DataFrame with all columns from ``df`` plus the
+ sampled UQ columns and a composite string ``sample`` column.
+ """
+ rng = np.random.default_rng(seed=seed)
+ uq = load_uq(uq_file)
+ n_samples = uq.samples
+
+ uq_df = generate_samples(uq.to_flat(), n_samples=n_samples, method=uq.method, seed=seed)
+
+ if posterior_file is not None:
+ posterior_df = pd.read_csv(posterior_file).drop(columns=["Unnamed: 0", "exp_id"], errors="ignore")
+ choice_indices = rng.choice(range(len(posterior_df)), n_samples)
+ posterior_sampled_df = posterior_df.iloc[choice_indices].reset_index(drop=True)
+ duplicate_cols = list(set(uq_df.columns) & set(posterior_sampled_df.columns) - {"sample"})
+ if duplicate_cols:
+ print(f"WARNING: posterior overrides UQ for columns: {sorted(duplicate_cols)}")
+ uq_df = uq_df.drop(columns=duplicate_cols)
+ uq_df = pd.concat([uq_df, posterior_sampled_df], axis=1)
+ uq_df.rename(columns={"sample": "uq"}).to_csv(output_path / "uq.csv", index=False)
+
+ if uq.mapping:
+ uq_df = apply_choice_mapping(uq_df, df, uq.mapping)
+
+ merged_df = df.merge(uq_df, how="cross", suffixes=("_df", "_uq"))
+ merged_df["sample"] = merged_df["sample_df"].astype(str) + "_uq_" + merged_df["sample_uq"].astype(int).astype(str)
+ merged_df = merged_df.drop(columns=["sample_df", "sample_uq"])
+ return merged_df
+
+
+def _run(*, kind: str) -> None:
+ """
+ Shared CLI body for forward and inverse runs.
+
+ Parses arguments, stages inputs, optionally builds an ensemble, then
+ renders one run script per member by calling ``_render__run``.
+ The two CLI entry points :func:`run_forward` and :func:`run_inverse` are
+ one-line wrappers around this function.
+
+ Parameters
+ ----------
+ kind : {"forward", "inverse"}
+ Which run script template to render. Selects the per-row worker
+ and decides whether to include ``inverse.file`` in the UQ dict.
+ """
+ if kind not in ("forward", "inverse"):
+ raise ValueError(f"kind must be 'forward' or 'inverse', got {kind!r}")
+ render = _render_forward_run if kind == "forward" else _render_inverse_run
+
+ parser = _build_cli_parser(
+ description=f"Stage RGI Glacier and render a {kind} run script (ensemble if UQ_FILE is given).",
+ supports_execute=True,
+ )
options = parser.parse_args()
force_overwrite = options.force_overwrite
@@ -560,100 +893,152 @@ def run_ensemble():
input_path = glacier_path / "input"
input_path.mkdir(parents=True, exist_ok=True)
+ staging_path = glacier_path / "staging"
+ staging_path.mkdir(parents=True, exist_ok=True)
output_path = glacier_path / "output"
output_path.mkdir(parents=True, exist_ok=True)
config_file = file_localizer(options.CONFIG_FILE, path / "config")
+ pism_config_cdl = file_localizer(options.pism_config_cdl, path / "config") if options.pism_config_cdl else None
template_file = file_localizer(options.TEMPLATE_FILE, path / "templates")
- uq_file = file_localizer(options.UQ_FILE, path / "uq")
+ uq_file = file_localizer(options.UQ_FILE, path / "uq") if options.UQ_FILE else None
- resolution = options.resolution
- posterior_file = options.posterior_file
- debug = options.debug
- queue = options.queue
- ntasks = options.ntasks
- nodes = options.nodes
- walltime = options.walltime
+ start_cli = options.start
+ end_cli = options.end
cfg = load_config(config_file)
+ # ``years`` is derived from the *effective* run span: CLI overrides win
+ # over the config's [time] section. Local Timestamps stay distinct from
+ # the CLI string overrides (``start_cli`` / ``end_cli``) below.
+ start_ts = pd.Timestamp(start_cli or cfg.time.time_start)
+ end_ts = pd.Timestamp(end_cli or cfg.time.time_end)
+ last_year = end_ts.year - 1 if (end_ts.month == 1 and end_ts.day == 1) else end_ts.year
+ years = list(range(start_ts.year, last_year + 1))
campaign_config = cfg.campaign.as_params()
- df = stage_glacier(campaign_config, rgi_id, path=input_path, force_overwrite=force_overwrite)
-
- seed = 42
- rng = np.random.default_rng(seed=seed)
- uq = load_uq(uq_file)
- n_samples = uq.samples
- mapping = uq.mapping
- uq_df = create_samples(uq.to_flat(), n_samples=n_samples, seed=seed)
- if posterior_file is not None:
- posterior_df = pd.read_csv(posterior_file).drop(columns=["Unnamed: 0", "exp_id"], errors="ignore")
- choice_indices = rng.choice(range(len(posterior_df)), n_samples)
- posterior_sampled_df = posterior_df.iloc[choice_indices].reset_index(drop=True)
- duplicate_cols = list(set(uq_df.columns) & set(posterior_sampled_df.columns) - {"sample"})
- if duplicate_cols:
- print(f"WARNING: posterior overrides UQ for columns: {sorted(duplicate_cols)}")
- uq_df = uq_df.drop(columns=duplicate_cols)
- uq_df = pd.concat([uq_df, posterior_sampled_df], axis=1)
+ campaign_config["years"] = years
+
+ df = stage_glacier(
+ campaign_config,
+ rgi_id,
+ path=input_path,
+ staging_path=staging_path,
+ force_overwrite=force_overwrite,
+ )
- uq_file = output_path / Path("uq.csv")
- uq_df.rename(columns={"sample": "id"}).to_csv(uq_file, index=False)
+ if uq_file is not None:
+ rows_df = _build_ensemble_df(df, uq_file, output_path, options.posterior_file)
+ header = f"Generate Ensemble Runs for Glacier {rgi_id}"
+ else:
+ rows_df = df
+ header = f"Generate Run for Glacier {rgi_id}"
+ is_ensemble = uq_file is not None
f = Figlet(font="standard")
banner = f.renderText("pism-terra")
- print("=" * 80)
+ print("=" * 120)
print(banner)
- print("=" * 80)
- print(f"Generate Ensemble Runs for Glacier {rgi_id}")
- print("-" * 80)
- if uq.mapping:
- uq_df = apply_choice_mapping(uq_df, df, uq.mapping)
- merged_df = df.merge(uq_df, how="cross", suffixes=("_df", "_uq"))
- df_columns = list(df.columns)
- for _, row in merged_df.iterrows():
- df_sample = row["sample_df"] if "sample_df" in row else ""
- uq_sample = str(int(row["sample_uq"])) if "sample_uq" in row else str(int(row["sample"]))
- sample = f"{df_sample}_uq_{uq_sample}" if df_sample else uq_sample
-
- outline_file = row["outline"]
- scalar_offset_file = input_path / Path(f"scalar_offset_{rgi_id}_id_{uq_sample}.nc")
+ print("=" * 120)
+ print(header)
+ print("-" * 120)
+
+ config_cli = {
+ "resolution": options.resolution,
+ "nodes": options.nodes,
+ "ntasks": options.ntasks,
+ "tasks": options.tasks,
+ "queue": options.queue,
+ "walltime": options.walltime,
+ "stress_balance": options.stress_balance,
+ "start": start_cli,
+ "end": end_cli,
+ }
+
+ for idx, row in rows_df.iterrows():
delta_T = row["atmosphere.delta_T"] if "atmosphere.delta_T" in row else 0
frac_P = row["atmosphere.frac_P"] if "atmosphere.frac_P" in row else 0
+ scalar_offset_file = input_path / Path(f"scalar_offset_{rgi_id}_id_{idx}.nc")
create_offset_file(scalar_offset_file, delta_T=delta_T, frac_P=frac_P)
- row_uq = row.drop(labels=df_columns + ["sample_df", "sample_uq"], errors="ignore").to_dict()
- row_uq.update(
+ if is_ensemble:
+ # Drop the staged-glacier columns and the composite sample id;
+ # whatever remains is a row of UQ overrides to forward to PISM.
+ uq_overrides = row.drop(labels=list(df.columns) + ["sample"]).to_dict()
+ else:
+ uq_overrides = {}
+
+ uq_overrides.update(
{
"input.file": row["boot_file"],
"grid.file": row["grid_file"],
- "atmosphere.given.file": row["climate_file"],
- "atmosphere.elevation_change.file": row["climate_file"],
"atmosphere.delta_T.file": scalar_offset_file,
- "atmosphere.frac_P.file": scalar_offset_file,
+ "atmosphere.elevation_change.file": row["boot_file"],
"atmosphere.precip_scaling.file": scalar_offset_file,
+ "atmosphere.given.file": row["climate_file"],
+ "energy.bedrock_thermal.file": row["heatflux_file"],
+ "surface.debm_simple.albedo_input.file": row["climate_file"],
+ "surface.debm_simple.std_dev.file": row["climate_file"],
"surface.force_to_thickness.file": row["boot_file"],
+ "surface.pdd.std_dev.file": row["climate_file"],
}
)
- run_glacier(
+ if kind == "inverse":
+ uq_overrides["inverse.file"] = row["obs_file"]
+
+ outline_file = row["outline_file"] if "outline_file" in row else None
+ # Keep numeric sample ids as ints (``id_0``) but preserve string period
+ # tags such as ``snap_1920_1949`` (``id_snap_1920_1949``).
+ if is_ensemble or "sample" not in row:
+ sample = row["sample"] if is_ensemble else idx
+ else:
+ try:
+ sample = int(row["sample"])
+ except (ValueError, TypeError):
+ sample = row["sample"]
+ render(
rgi_id,
config_file,
template_file,
outline_file,
path=path,
- resolution=resolution,
- nodes=nodes,
- ntasks=ntasks,
- queue=queue,
- walltime=walltime,
- debug=debug,
- uq=row_uq,
+ config_cli=config_cli,
+ debug=options.debug,
+ uq=uq_overrides,
sample=sample,
+ pism_config_cdl=pism_config_cdl,
)
+ # ``--execute`` only fires the *first* generated script. Use it for the
+ # single-glacier mode; ignore it for ensembles (would only run member 0).
+ if not is_ensemble and getattr(options, "execute", False) and not options.debug:
+ find_first_and_execute(path / rgi_id)
+
if options.bucket:
prefix = f"{options.bucket_prefix}/{rgi_id}" if options.bucket_prefix else rgi_id
local_to_s3(glacier_path, bucket=options.bucket, prefix=prefix)
+def run_forward() -> None:
+ """
+ CLI entry point for forward runs (single or ensemble).
+
+ Behaves as a single-glacier run when no ``UQ_FILE`` positional is
+ supplied, and as a UQ ensemble when one is. The argument schema and
+ output layout are otherwise identical.
+ """
+ _run(kind="forward")
+
+
+def run_inverse() -> None:
+ """
+ CLI entry point for inverse runs (single or ensemble).
+
+ Behaves as a single-glacier inverse run when no ``UQ_FILE`` positional
+ is supplied, and as a UQ ensemble when one is. The per-row UQ dict
+ additionally maps ``inverse.file`` to the staged observation file.
+ """
+ _run(kind="inverse")
+
+
if __name__ == "__main__":
__spec__ = None # type: ignore
- run_single()
+ run_forward()
diff --git a/pism_terra/glacier/s4f.py b/pism_terra/glacier/s4f.py
new file mode 100644
index 0000000..e719ca3
--- /dev/null
+++ b/pism_terra/glacier/s4f.py
@@ -0,0 +1,341 @@
+# Copyright (C) 2025 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+# pylint: disable=unused-import,unused-variable,broad-exception-caught,too-many-positional-arguments
+
+"""
+S4F Mission Planning.
+
+Methods to assist with Snow4Flow Mission Planning.
+"""
+
+import time
+from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Callable
+
+import cf_xarray
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+import pyogrio
+import rioxarray
+import xarray as xr
+from pyfiglet import Figlet
+from shapely.geometry import Point, Polygon, box
+
+from pism_terra.aws import download_from_s3, local_to_s3
+from pism_terra.config import load_config
+from pism_terra.domain import create_domain, get_bounds_from_geometry
+from pism_terra.glacier.climate import (
+ create_offset_file,
+ create_step_file,
+ era5,
+ snap,
+)
+from pism_terra.glacier.dem import boot_file_from_grid
+from pism_terra.raster import apply_perimeter_band
+from pism_terra.vector import (
+ get_glacier_from_rgi_id,
+ glaciers_in_complex,
+ grid_cells_from_dataset,
+ grid_points_from_dataset,
+)
+from pism_terra.workflow import check_dataset_fully, check_xr_fully, check_xr_lazy
+
+xr.set_options(keep_attrs=True)
+
+CLIMATE: Mapping[str, Callable] = {"era5": era5, "snap": snap}
+MODIFIER: Mapping[str, Callable] = {
+ "era5": create_offset_file,
+ "snap": create_offset_file,
+}
+
+
+def main():
+ """
+ Run main script.
+ """
+
+ # set up the option parser
+ parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+ parser.description = "Stage RGI Glacier."
+ parser.add_argument("--bucket", help="AWS S3 Bucket to upload output files to")
+ parser.add_argument(
+ "--bucket-prefix",
+ help="AWS prefix (location in bucket) to add to product files",
+ default="",
+ )
+ parser.add_argument(
+ "--output-path",
+ help="Path to save all files.",
+ type=Path,
+ default=Path("data"),
+ )
+ parser.add_argument(
+ "--force-overwrite",
+ help="Force downloading all files.",
+ action="store_true",
+ default=False,
+ )
+ parser.add_argument(
+ "CONFIG_FILE",
+ help="CONFIG TOML.",
+ nargs=1,
+ )
+
+ options, unknown = parser.parse_known_args()
+ path = options.output_path
+ config_file = options.CONFIG_FILE[0]
+ force_overwrite = options.force_overwrite
+
+ path.mkdir(parents=True, exist_ok=True)
+
+ cfg = load_config(config_file)
+ config = cfg.campaign.as_params()
+
+ print("RGI Database")
+ rgi_s3_uri = f"""s3://{config["bucket"]}/{config["prefix"]}/rgi/{config["rgi_complex_file"]}"""
+ rgi_local = path / config["rgi_complex_file"]
+ if not rgi_local.exists():
+ print(f"Downloading {rgi_s3_uri} -> {rgi_local}")
+ download_from_s3(rgi_s3_uri, rgi_local)
+ else:
+ print(f"Using cached {rgi_local}")
+ rgi = gpd.read_file(rgi_local)
+
+ rgi_cloud = path / config["rgi_complex_file"].replace("gpkg", "fgb")
+ rgi.to_file(rgi_cloud)
+
+ all_nc_files: list[Path] = []
+ for rgi_id in rgi.rgi_id:
+ glacier_path = path / Path(rgi_id)
+ glacier_path.mkdir(parents=True, exist_ok=True)
+
+ input_path = glacier_path / Path("input")
+ input_path.mkdir(parents=True, exist_ok=True)
+ staging_path = glacier_path / Path("staging")
+ staging_path.mkdir(parents=True, exist_ok=True)
+ glacier_boot_files = s4f_glacier(
+ config,
+ rgi_id,
+ path=input_path,
+ staging_path=staging_path,
+ force_overwrite=force_overwrite,
+ )
+ all_nc_files.extend(Path(p) for p in glacier_boot_files.values() if Path(p).suffix == ".nc")
+
+ total_bytes = sum(p.stat().st_size for p in all_nc_files if p.exists())
+ if total_bytes >= 1 << 30:
+ size = f"{total_bytes / (1 << 30):.2f} GB"
+ elif total_bytes >= 1 << 20:
+ size = f"{total_bytes / (1 << 20):.2f} MB"
+ else:
+ size = f"{total_bytes / (1 << 10):.2f} KB"
+ print(f"Total size of {len(all_nc_files)} NetCDF files: {size}")
+
+ bucket = config["bucket"]
+ prefix = config["prefix"]
+ print("Now run")
+ print(f"""aws s3 sync {path} s3://{bucket}/{prefix}/planning --exclude "*/staging/*" """)
+
+
+def s4f_glacier(
+ config: dict,
+ rgi_id: str,
+ path: str | Path = "input_files",
+ staging_path: str | Path | None = None,
+ resolution: float = 100.0,
+ force_overwrite: bool = False,
+) -> dict:
+ """
+ Stage glacier inputs (boot dataset and Cloud Optimized GeoTIFFs).
+
+ For the glacier identified by ``rgi_id``, this function:
+ (1) loads the glacier geometry (downloading the RGI vector file from S3
+ if needed),
+ (2) builds a DEM/thickness/bed/velocity “boot” dataset via
+ :func:`boot_file_from_grid`,
+ (3) creates a target model grid,
+ (4) writes one Cloud Optimized GeoTIFF per spatial variable in the boot
+ dataset (plus a NetCDF copy of ``bed`` and a clipped surface tif),
+ and (5) returns a mapping of variable identifiers to written file paths.
+
+ Parameters
+ ----------
+ config : dict
+ Configuration mapping. Must contain at least:
+
+ - ``"bucket"`` and ``"prefix"`` : str — S3 location for the RGI files.
+ - ``"rgi_complex_file"`` : str — RGI glacier-complex ("-C") filename.
+ - ``"rgi_glacier_file"`` : str — RGI glacier ("-G") filename.
+ - ``"dem"`` : str — DEM source passed to :func:`boot_file_from_grid`.
+ - ``"ice_thickness"`` : str — ice thickness source.
+ - ``"velocity"`` : str — velocity source.
+ rgi_id : str
+ Glacier identifier (e.g., ``"RGI2000-v7.0-C-06-00014"``).
+ path : str or pathlib.Path, default ``"input_files"``
+ Final output directory. Created if missing. Holds only the
+ Cloud Optimized GeoTIFF outputs (one per spatial variable in the boot
+ dataset).
+ staging_path : str or pathlib.Path or None, optional
+ Working directory for intermediate files (RGI table cache, glacier
+ outline FlatGeobuf, DEM tifs, ice-thickness/velocity intermediates).
+ Created if missing. If ``None`` (default), falls back to ``path``
+ (legacy behavior — everything in one directory).
+ resolution : float, default ``100.0``
+ Target grid resolution (meters), used both for grid construction and in
+ output filenames.
+ force_overwrite : bool, default ``False``
+ If ``True``, downstream helpers may regenerate intermediate/final artifacts
+ even if cache files exist (e.g., passed to :func:`boot_file_from_grid`).
+
+ Returns
+ -------
+ dict
+ Mapping from variable identifier ``f"{rgi_id}_{var}"`` to the absolute
+ path of the written Cloud Optimized GeoTIFF (or NetCDF, in the case of
+ ``bed``).
+
+ Raises
+ ------
+ KeyError
+ If required keys are missing in ``config``.
+ ValueError
+ If ``rgi_id`` is not found in the RGI layer or geometry/CRS is invalid.
+ Exception
+ Propagated errors from DEM/thickness preparation, reprojection, or I/O.
+
+ See Also
+ --------
+ boot_file_from_grid
+ Builds the boot (DEM, thickness, bed, masks) dataset around the glacier.
+ create_domain
+ Creates the target model grid and bounds.
+
+ Notes
+ -----
+ - Writes the complex outline as ``rgi_{rgi_id}-C.gpkg`` and the member
+ glacier outlines as ``rgi_{rgi_id}-G.gpkg`` in ``staging_path``.
+ """
+
+ f = Figlet(font="standard")
+ banner = f.renderText("pism-terra")
+ print("=" * 120)
+ print(banner)
+ print("=" * 120)
+ print(f"S4F Planning Glacier {rgi_id}")
+ print("-" * 120)
+ print("")
+
+ # Output dirs: `path` holds only COGs; `staging_path` holds intermediates.
+ path = Path(path)
+ path.mkdir(parents=True, exist_ok=True)
+ staging_path = Path(staging_path) if staging_path is not None else path
+ staging_path.mkdir(parents=True, exist_ok=True)
+
+ print("RGI Database")
+ rgi_complex_local = staging_path / config["rgi_complex_file"]
+ if not rgi_complex_local.exists():
+ rgi_complex_s3_uri = f"""s3://{config["bucket"]}/{config["prefix"]}/rgi/{config["rgi_complex_file"]}"""
+ print(f"Downloading {rgi_complex_s3_uri} -> {rgi_complex_local}")
+ download_from_s3(rgi_complex_s3_uri, rgi_complex_local)
+ else:
+ print(f"Using cached {rgi_complex_local}")
+
+ rgi_glacier_local = staging_path / config["rgi_glacier_file"]
+ if not rgi_glacier_local.exists():
+ rgi_glacier_s3_uri = f"""s3://{config["bucket"]}/{config["prefix"]}/rgi/{config["rgi_glacier_file"]}"""
+ print(f"Downloading {rgi_glacier_s3_uri} -> {rgi_glacier_local}")
+ download_from_s3(rgi_glacier_s3_uri, rgi_glacier_local)
+ else:
+ print(f"Using cached {rgi_glacier_local}")
+
+ # NOTE: gpd.read_file/to_file corrupts the heap on some envs and crashes the
+ # next libgdal allocation (e.g. inside dem_stitcher). Use pyogrio directly.
+ rgi_complex = pyogrio.read_dataframe(rgi_complex_local, use_arrow=False)
+ glacier = get_glacier_from_rgi_id(rgi_complex, rgi_id)
+ if glacier.empty:
+ raise ValueError(f"RGI ID not found: {rgi_id}")
+
+ dst_crs = glacier["crs"].values[0]
+ glacier_projected = glacier.to_crs(dst_crs)
+
+ # Write the complex ("-C") and member-glacier ("-G") outlines (mirrors stage.py).
+ glacier_complex_file = staging_path / f"rgi_{rgi_id}-C.gpkg"
+ pyogrio.write_dataframe(glacier, glacier_complex_file)
+
+ glacier_file = staging_path / f"rgi_{rgi_id}-G.gpkg"
+ rgi_glacier = pyogrio.read_dataframe(rgi_glacier_local, use_arrow=False)
+ glacier_ids = glaciers_in_complex(rgi_id, rgi_glacier)
+ glaciers = rgi_glacier[rgi_glacier["rgi_id"].isin(glacier_ids)]
+ if glaciers.empty:
+ print(f"Warning: no glacier outlines found for complex {rgi_id}")
+ pyogrio.write_dataframe(glaciers, glacier_file)
+
+ x_bnds, y_bnds = get_bounds_from_geometry(glacier_projected.geometry, buffer_dist=2_000.0, dx=1_000.0)
+ grid_ds = create_domain(x_bnds, y_bnds, resolution=resolution, crs=dst_crs)
+
+ # Build boot dataset (DEM/thickness/bed) — caches go to staging
+ boot_ds = boot_file_from_grid(
+ grid_ds,
+ rgi_id,
+ glacier_projected.geometry,
+ dem_dataset=config["dem"],
+ ice_thickness_dataset=config["ice_thickness"],
+ velocity_dataset=config["velocity"],
+ bathymetry_dataset=config["bathymetry"],
+ forcing_mask=config["forcing_mask"] if config["forcing_mask"] else None,
+ path=staging_path,
+ force_overwrite=force_overwrite,
+ bucket=config["bucket"],
+ prefix=config["prefix"],
+ )
+
+ print("")
+ print("Saving Cloud Optimized GeoTIFFs")
+ print("-" * 120)
+ boot_files = {}
+ for var in boot_ds.data_vars:
+ da = boot_ds[var]
+ if not {"x", "y"}.issubset(set(da.dims)):
+ continue
+ m_id = f"{rgi_id}_{var}"
+ cog_path = path / f"{m_id}.tif"
+ out = da.astype("uint8") if da.dtype == bool else da
+ out.rio.to_raster(cog_path, driver="COG", compress="DEFLATE")
+ print(cog_path)
+ boot_files[m_id] = cog_path
+ if var == "surface":
+ cog_clipped_path = path / f"{m_id}_clipped.tif"
+ out_clipped = out.rio.clip(glacier_projected.geometry, drop=False)
+ out_clipped.rio.to_raster(cog_clipped_path, driver="COG", compress="DEFLATE")
+ print(cog_clipped_path)
+ if var == "bed":
+ encoding = {var: {"zlib": True, "complevel": 2, "shuffle": True}}
+ nc_path = path / f"{m_id}.nc"
+ out.to_netcdf(nc_path, encoding=encoding, engine="h5netcdf")
+ print(nc_path)
+ boot_files[m_id] = nc_path
+ return boot_files
+
+
+if __name__ == "__main__":
+ __spec__ = None # type: ignore
+ main()
diff --git a/pism_terra/glacier/stage.py b/pism_terra/glacier/stage.py
index d045f8a..40feb65 100644
--- a/pism_terra/glacier/stage.py
+++ b/pism_terra/glacier/stage.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2025 Andy Aschwanden
+# Copyright (C) 2025-26 Andy Aschwanden
#
# This file is part of pism-terra.
#
@@ -21,6 +21,8 @@
Staging.
"""
+import re
+import shutil
import time
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from collections.abc import Mapping
@@ -30,31 +32,60 @@
import cf_xarray
import geopandas as gpd
+import numpy as np
import pandas as pd
+import pyogrio
import rioxarray
import xarray as xr
from pyfiglet import Figlet
-from shapely.geometry import Polygon
+from shapely.geometry import Point, Polygon, box
from pism_terra.aws import download_from_s3, local_to_s3
from pism_terra.config import load_config
-from pism_terra.domain import create_grid
-from pism_terra.glacier.climate import create_offset_file, era5, pmip4, snap
-from pism_terra.glacier.dem import boot_file_from_rgi_id
+from pism_terra.domain import create_domain, get_bounds_from_geometry
+from pism_terra.glacier.climate import (
+ carra2,
+ create_offset_file,
+ create_step_file,
+ era5,
+ era5_mean,
+ era5_monthly_mean,
+ snap,
+)
+from pism_terra.glacier.dem import boot_file_from_grid
+from pism_terra.glacier.observations import glacier_velocities_from_grid
+from pism_terra.heatflux import heatflux_from_grid
from pism_terra.raster import apply_perimeter_band
-from pism_terra.vector import get_glacier_from_rgi_id
-from pism_terra.workflow import check_dataset_fully, check_xr_fully, check_xr_lazy
+from pism_terra.vector import get_glacier_from_rgi_id, glaciers_in_complex
+from pism_terra.workflow import (
+ check_dataset_fully,
+ check_xr_fully,
+ check_xr_lazy,
+ drop_geotransform_attr,
+ stamp_grid_mapping,
+)
xr.set_options(keep_attrs=True)
-CLIMATE: Mapping[str, Callable] = {"pmip4": pmip4, "era5": era5, "snap": snap}
+CLIMATE: Mapping[str, Callable] = {
+ "carra2": carra2,
+ "era5": era5,
+ "era5-mean": era5_mean,
+ "era5-monthly-mean": era5_monthly_mean,
+ "snap-monthly-mean": snap,
+}
+MODIFIER: Mapping[str, Callable] = {
+ "era5": create_offset_file,
+ "snap": create_offset_file,
+}
def stage_glacier(
config: dict,
rgi_id: str,
path: str | Path = "input_files",
- resolution: float = 50.0,
+ staging_path: str | Path | None = None,
+ resolution: float = 100.0,
force_overwrite: bool = False,
) -> pd.DataFrame:
"""
@@ -73,27 +104,35 @@ def stage_glacier(
config : dict
Configuration mapping. Must contain at least:
- ``"dem"`` : str
- DEM source passed to :func:`boot_file_from_rgi_id`.
+ DEM source passed to :func:`boot_file_from_grid`.
- ``"climate"`` : str
Key in :data:`CLIMATE` (e.g., ``"pmip4"``) selecting the climate builder.
rgi_id : str
Glacier identifier (e.g., ``"RGI2000-v7.0-C-06-00014"``).
path : str or pathlib.Path, default ``"input_files"``
- Output directory. Created if missing. All staged artifacts are written here.
- resolution : float, default ``50.0``
+ Final output directory. Created if missing. Holds the artifacts that
+ downstream tooling consumes: glacier outline GPKG, boot NetCDF,
+ grid NetCDF, and climate forcing NetCDF.
+ staging_path : str or pathlib.Path or None, optional
+ Working directory for intermediate files (RGI table cache, DEM tifs,
+ ice-thickness/velocity intermediates, ERA5/PMIP4 raw downloads,
+ debug GPKGs, domain-bounds polygon). Created if missing. If ``None``
+ (default), falls back to ``path`` (legacy behavior — everything in
+ one directory).
+ resolution : float, default ``100.0``
Target grid resolution (meters), used both for grid construction and in
output filenames.
force_overwrite : bool, default ``False``
If ``True``, downstream helpers may regenerate intermediate/final artifacts
- even if cache files exist (e.g., passed to :func:`boot_file_from_rgi_id`
+ even if cache files exist (e.g., passed to :func:`boot_file_from_grid`
and to the selected climate builder via :data:`CLIMATE`).
Returns
-------
pandas.DataFrame
One row per produced **climate** file, with absolute-path columns:
- ``rgi_id``, ``outline`` (GPKG), ``boot_file`` (NetCDF),
- ``grid_file`` (NetCDF), ``climate_file`` (NetCDF).
+ ``rgi_id``, ``outline_file`` (GPKG), ``boot_file`` (NetCDF),
+ ``grid_file`` (NetCDF), ``climate_file`` (NetCDF), and ``sample`` (int).
Raises
------
@@ -108,109 +147,130 @@ def stage_glacier(
See Also
--------
- boot_file_from_rgi_id
+ boot_file_from_grid
Builds the boot (DEM, thickness, bed, masks) dataset around the glacier.
- create_grid
+ create_domain
Creates the target model grid and bounds.
CLIMATE
Mapping from climate name (e.g., ``"pmip4"``) to a function that generates
climate NetCDF file(s) for the glacier domain.
-
- Notes
- -----
- - Applies :func:`apply_perimeter_band` to clean DEM edges.
- - Enforces simple constraints (non-negative thickness; bed below surface).
- - Writes two vector layers:
- - Glacier outline: ``rgi_{rgi_id}.gpkg`` (same CRS as RGI entry).
- - Domain bounds polygon: ``domain_{rgi_id}.gpkg``.
- - The returned DataFrame is convenient for downstream orchestration/fan-out.
"""
f = Figlet(font="standard")
banner = f.renderText("pism-terra")
- print("=" * 80)
+ print("=" * 120)
print(banner)
- print("=" * 80)
+ print("=" * 120)
print(f"Stage Glacier {rgi_id}")
- print("-" * 80)
+ print("-" * 120)
print("")
- # Outputs dir
+ # Output dirs: `path` holds final artifacts; `staging_path` holds intermediates.
path = Path(path)
path.mkdir(parents=True, exist_ok=True)
+ staging_path = Path(staging_path) if staging_path is not None else path
+ staging_path.mkdir(parents=True, exist_ok=True)
print("RGI Database")
- rgi_s3_uri = f"""s3://{config["bucket"]}/{config["prefix"]}/rgi/{config["rgi_file"]}"""
- rgi_local = path / config["rgi_file"]
- if not rgi_local.exists():
- print(f"Downloading {rgi_s3_uri} -> {rgi_local}")
- download_from_s3(rgi_s3_uri, rgi_local)
+ rgi_glacier_s3_uri = f"""s3://{config["bucket"]}/{config["prefix"]}/rgi/{config["rgi_glacier_file"]}"""
+ rgi_glacier_local = staging_path / config["rgi_glacier_file"]
+ if not rgi_glacier_local.exists():
+ print(f"Downloading {rgi_glacier_s3_uri} -> {rgi_glacier_local}")
+ download_from_s3(rgi_glacier_s3_uri, rgi_glacier_local)
+ else:
+ print(f"Using cached {rgi_glacier_local}")
+
+ rgi_complex_s3_uri = f"""s3://{config["bucket"]}/{config["prefix"]}/rgi/{config["rgi_complex_file"]}"""
+ rgi_complex_local = staging_path / config["rgi_complex_file"]
+ if not rgi_complex_local.exists():
+ print(f"Downloading {rgi_complex_s3_uri} -> {rgi_complex_local}")
+ download_from_s3(rgi_complex_s3_uri, rgi_complex_local)
else:
- print(f"Using cached {rgi_local}")
- rgi = gpd.read_file(rgi_local)
+ print(f"Using cached {rgi_complex_local}")
- glacier = get_glacier_from_rgi_id(rgi, rgi_id)
+ # NOTE: gpd.read_file/to_file (via pyogrio's geopandas wrapper) corrupts the
+ # heap on some envs and crashes the next libgdal allocation (e.g. inside
+ # dem_stitcher). Calling pyogrio directly avoids the trigger.
+ rgi_complex = pyogrio.read_dataframe(rgi_complex_local, use_arrow=False)
+ glacier = get_glacier_from_rgi_id(rgi_complex, rgi_id)
if glacier.empty:
raise ValueError(f"RGI ID not found: {rgi_id}")
- glacier_file = path / f"rgi_{rgi_id}.gpkg"
- glacier_series = glacier.iloc[0]
- crs = glacier_series["epsg"]
- glacier.to_file(glacier_file)
+ glacier_complex_file = path / f"rgi_{rgi_id}-C.gpkg"
+ dst_crs = glacier["crs"].values[0]
+ glacier_projected = glacier.to_crs(dst_crs)
+ pyogrio.write_dataframe(glacier, glacier_complex_file)
+
+ # Extract the individual glacier outlines that make up this complex and
+ # write them to the "-G" file. Membership is given by the glacier-level
+ # "rgi_id_c" column (handled by glaciers_in_complex, incl. aggregates).
+ glacier_file = path / f"rgi_{rgi_id}-G.gpkg"
+ rgi_glacier = pyogrio.read_dataframe(rgi_glacier_local, use_arrow=False)
+ glacier_ids = glaciers_in_complex(rgi_id, rgi_glacier)
+ glaciers = rgi_glacier[rgi_glacier["rgi_id"].isin(glacier_ids)]
+ if glaciers.empty:
+ print(f"Warning: no glacier outlines found for complex {rgi_id}")
+ pyogrio.write_dataframe(glaciers, glacier_file)
+
+ x_bnds, y_bnds = get_bounds_from_geometry(glacier_projected.geometry, buffer_dist=5_000.0, dx=1_000.0)
+ grid_ds = create_domain(x_bnds, y_bnds, resolution=resolution, crs=dst_crs)
# Output filenames
- boot_file = path / f"bootfile_g{int(resolution)}m_{rgi_id}.nc"
- grid_file = path / f"grid_g{int(resolution)}m_{rgi_id}.nc"
-
- # Build boot dataset (DEM/thickness/bed)
- boot_ds = boot_file_from_rgi_id(
+ boot_file = path / f"bootfile_{rgi_id}.nc"
+ grid_file = path / f"grid_{rgi_id}.nc"
+ obs_file = path / f"obs_{rgi_id}.nc"
+ bheatflux_file = path / f"bheatflux_{rgi_id}.nc"
+
+ # Build boot dataset (DEM/thickness/bed) — caches go to staging
+ boot_ds = boot_file_from_grid(
+ grid_ds,
rgi_id,
- rgi,
+ glacier_projected.geometry,
dem_dataset=config["dem"],
ice_thickness_dataset=config["ice_thickness"],
velocity_dataset=config["velocity"],
- buffer_distance=10000.0,
- path=path,
+ bathymetry_dataset=config["bathymetry"],
+ forcing_mask=config["forcing_mask"],
+ path=staging_path,
force_overwrite=force_overwrite,
bucket=config["bucket"],
+ prefix=config["prefix"],
)
- # Grid & bounds
- grid_ds = create_grid(glacier, boot_ds, crs=crs, buffer_distance=8000.0)
- bounds = [
- grid_ds["x_bnds"].values[0][0],
- grid_ds["y_bnds"].values[0][0],
- grid_ds["x_bnds"].values[0][1],
- grid_ds["y_bnds"].values[0][1],
- ]
-
- # Edge cleanup and simple physical constraints
- for v in ["bed"]:
- boot_ds[v] = apply_perimeter_band(boot_ds[v], bounds=bounds)
- for v in ["surface"]:
- boot_ds[v] = apply_perimeter_band(boot_ds[v], bounds=bounds, value=0.0)
- boot_ds["thickness"] = boot_ds["thickness"].where(boot_ds["thickness"] > 0.0, 0.0)
- boot_ds.rio.write_crs(crs, inplace=True)
- if hasattr(boot_ds["spatial_ref"], "GeoTransform"):
- del boot_ds["spatial_ref"].attrs["GeoTransform"]
- for name in ("x", "y", "thickness", "bed", "surface", "tillwat", "ftt_mask", "land_ice_area_fraction_retreat"):
- if name in boot_ds:
- boot_ds[name].encoding.update({"_FillValue": None})
- boot_ds.rio.write_coordinate_system(inplace=True)
-
print("")
print("Saving bootfile")
- print("-" * 80)
+ print("-" * 120)
boot_file.unlink(missing_ok=True)
- boot_ds.to_netcdf(boot_file, engine="netcdf4")
+ # rioxarray writes a GeoTransform on spatial_ref consistent with ascending y
+ # (positive dy); GDAL then prefers that over the y coordinate variable and
+ # displays the raster upside-down in QGIS. Drop it so GDAL falls back to
+ # deriving the transform from the y coordinate variable (top-down).
+ drop_geotransform_attr(boot_ds)
+ boot_ds = stamp_grid_mapping(boot_ds)
+ boot_ds.to_netcdf(boot_file, engine="h5netcdf")
check_xr_lazy(boot_file)
+ heatflux_ds = heatflux_from_grid(
+ grid_ds,
+ dataset=config["heatflux"],
+ path=bheatflux_file,
+ bucket=config["bucket"],
+ prefix=config["prefix"],
+ force_overwrite=force_overwrite,
+ )
+ check_xr_fully(bheatflux_file)
+
grid_ds.attrs.update({"domain": rgi_id})
grid_file.unlink(missing_ok=True)
- grid_ds.to_netcdf(grid_file, engine="netcdf4")
+ drop_geotransform_attr(grid_ds)
+ grid_ds = stamp_grid_mapping(grid_ds)
+ grid_ds.to_netcdf(grid_file, engine="h5netcdf")
check_xr_fully(grid_file)
- # Save domain extent polygon as a GPKG
+ _ = glacier_velocities_from_grid(grid_ds, glacier_projected.geometry, path=obs_file)
+ check_xr_fully(obs_file)
+
+ # Save domain extent polygon as a GPKG (intermediate, used for sanity checks)
x_point_list = [
grid_ds.x_bnds[0][0],
grid_ds.x_bnds[0][0],
@@ -226,33 +286,60 @@ def stage_glacier(
grid_ds.y_bnds[0][0],
]
domain_bounds_geom = Polygon(zip(x_point_list, y_point_list))
- domain_bounds = gpd.GeoDataFrame(index=[0], crs=crs, geometry=[domain_bounds_geom])
- domain_bounds_file = path / f"domain_{rgi_id}.gpkg"
- domain_bounds.to_file(domain_bounds_file)
+ domain_bounds = gpd.GeoDataFrame(index=[0], crs=dst_crs, geometry=[domain_bounds_geom])
+ domain_bounds_file = staging_path / f"domain_{rgi_id}.gpkg"
+ pyogrio.write_dataframe(domain_bounds, domain_bounds_file)
- scalar_offset_file = path / Path(f"scalar_offset_{rgi_id}_id_0.nc")
- create_offset_file(scalar_offset_file, delta_T=0.0, frac_P=0.0)
-
- # Climate forcing
+ clim_mod = config["climate"]
+ # Climate forcing — built into staging, then final outputs moved to `path`
climate_from_rgi = CLIMATE[config["climate"]]
- responses = climate_from_rgi(rgi_id=rgi_id, rgi=rgi, path=path, force_overwrite=force_overwrite) # list[Path]
+ responses = climate_from_rgi(
+ grid_ds,
+ rgi_id=rgi_id,
+ years=config["years"],
+ path=staging_path,
+ bucket=config["bucket"],
+ prefix=config["prefix"],
+ force_overwrite=force_overwrite,
+ ) # list[Path]
# Normalize to list[Path]
if isinstance(responses, (str, Path)):
responses = [Path(responses)]
else:
responses = [Path(p) for p in responses]
+ if staging_path.resolve() != path.resolve():
+ # Copy (not move) the final climate file(s) into the input dir, leaving
+ # the builder's output in staging so its own cache guard short-circuits
+ # on reruns. Moving deletes the staging copy, so the climate would be
+ # regenerated/re-downloaded every run even though it already exists in
+ # the input dir. (Staging is excluded from the S3 upload.)
+ copied: list[Path] = []
+ for src in responses:
+ dst = path / src.name
+ if src.resolve() != dst.resolve():
+ dst.unlink(missing_ok=True)
+ shutil.copy2(str(src), str(dst))
+ copied.append(dst)
+ responses = copied
# Build file index (one row per climate file)
files_dict = {
"rgi_id": rgi_id,
- "outline": glacier_file.resolve(),
+ "outline_file": glacier_file.resolve(),
"boot_file": boot_file.resolve(),
"grid_file": grid_file.resolve(),
- "scalar_offset_file": scalar_offset_file.resolve(),
+ "heatflux_file": bheatflux_file.resolve(),
+ "obs_file": obs_file.resolve(),
}
dfs: list[pd.DataFrame] = []
- for fpath in responses:
- row = {**files_dict, "climate_file": Path(fpath).resolve()}
+ for idx, fpath in enumerate(responses):
+ # When the climate source emits period-tagged files (e.g. SNAP's
+ # ``snap_1920_1949_.nc``), use that tag as the sample id so the
+ # run id carries the period (``id_snap_1920_1949``) and composes with a
+ # UQ file (``id_snap_1920_1949_uq_0``). Otherwise fall back to the index.
+ m = re.search(r"snap_\d{4}_\d{4}", Path(fpath).stem)
+ sample: str | int = m.group(0) if m else idx
+ row = {**files_dict, "climate_file": Path(fpath).resolve(), "sample": sample}
dfs.append(pd.DataFrame.from_dict([row]))
df = pd.concat(dfs).reset_index(drop=True)
@@ -277,7 +364,7 @@ def main():
"--output-path",
help="Path to save all files.",
type=Path,
- default=Path("data"),
+ default=Path("."),
)
parser.add_argument(
"--force-overwrite",
@@ -303,7 +390,15 @@ def main():
rgi_id = options.RGI_ID[0]
cfg = load_config(config_file)
+ # Cover every calendar year touched by the simulation. PISM's time.end is
+ # exclusive (e.g. "2025-01-01" means stop at midnight Jan 1, so 2025
+ # itself is not simulated), hence the - 1 when end is exactly Jan 1.
+ start = pd.Timestamp(cfg.time.time_start)
+ end = pd.Timestamp(cfg.time.time_end)
+ last_year = end.year - 1 if (end.month == 1 and end.day == 1) else end.year
+ years = list(range(start.year, last_year + 1))
config = cfg.campaign.as_params()
+ config["years"] = years
path.mkdir(parents=True, exist_ok=True)
glacier_path = path / Path(rgi_id)
@@ -311,10 +406,13 @@ def main():
input_path = glacier_path / Path("input")
input_path.mkdir(parents=True, exist_ok=True)
+ staging_path = glacier_path / Path("staging")
+ staging_path.mkdir(parents=True, exist_ok=True)
glacier_df = stage_glacier(
config,
rgi_id,
path=input_path,
+ staging_path=staging_path,
force_overwrite=force_overwrite,
)
glacier_df.to_csv(input_path / Path(f"{rgi_id}.csv"))
diff --git a/pism_terra/grids.py b/pism_terra/grids.py
new file mode 100644
index 0000000..610f5e9
--- /dev/null
+++ b/pism_terra/grids.py
@@ -0,0 +1,41 @@
+# Copyright (C) 2026 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+"""CDO grid description files for common projections."""
+
+from pathlib import Path
+
+_GRIDS_DIR = Path(__file__).parent / "grids"
+
+
+def load_grid(name: str) -> str:
+ """
+ Load a CDO grid description by name.
+
+ Parameters
+ ----------
+ name : str
+ Grid name (e.g., ``"ismip6"``, ``"carra2"``, ``"hirham"``, ``"bedmachine"``).
+ The ``.txt`` extension is appended automatically.
+
+ Returns
+ -------
+ str
+ Contents of the grid description file.
+ """
+ return (_GRIDS_DIR / f"{name}.txt").read_text(encoding="utf-8")
diff --git a/pism_terra/grids/bedmachine.txt b/pism_terra/grids/bedmachine.txt
new file mode 100644
index 0000000..848b94a
--- /dev/null
+++ b/pism_terra/grids/bedmachine.txt
@@ -0,0 +1,16 @@
+gridtype = projection
+xsize = 10218
+ysize = 18346
+xunits = "meter"
+yunits = "meter"
+xfirst = -65300
+xinc = 150
+yfirst = -3384425
+yinc = 150
+grid_mapping = crs
+grid_mapping_name = polar_stereographic
+straight_vertical_longitude_from_pole = -39.
+standard_parallel = 71.
+latitude_of_projection_origin = 90.
+false_easting = 0.
+false_northing = 0.
diff --git a/pism_terra/grids/carra2.txt b/pism_terra/grids/carra2.txt
new file mode 100644
index 0000000..0f33de1
--- /dev/null
+++ b/pism_terra/grids/carra2.txt
@@ -0,0 +1,17 @@
+gridtype = projection
+xsize = 2869
+ysize = 2869
+xunits = "meter"
+yunits = "meter"
+xfirst = -3585000
+xinc = 2500
+yfirst = -3585000
+yinc = 2500
+grid_mapping = crs
+grid_mapping_name = polar_stereographic
+straight_vertical_longitude_from_pole = -30.
+standard_parallel = 90.
+latitude_of_projection_origin = 90.
+earth_radius = 6371229.
+false_easting = 172840.374543307
+false_northing = 645049.059394855
diff --git a/pism_terra/grids/hirham.txt b/pism_terra/grids/hirham.txt
new file mode 100644
index 0000000..60b1538
--- /dev/null
+++ b/pism_terra/grids/hirham.txt
@@ -0,0 +1,15 @@
+gridtype = projection
+xname = rlon
+xsize = 402
+ysize = 602
+xfirst = -16
+xinc = 0.05
+xunits = degree
+yname = rlat
+yfirst = -14
+yinc = 0.05
+yunits = degree
+grid_mapping_name = rotated_latitude_longitude
+grid_north_pole_longitude = 160
+grid_north_pole_latitude = 18
+north_pole_grid_longitude = 0
diff --git a/pism_terra/grids/ismip6.txt b/pism_terra/grids/ismip6.txt
new file mode 100644
index 0000000..69ebb63
--- /dev/null
+++ b/pism_terra/grids/ismip6.txt
@@ -0,0 +1,16 @@
+gridtype = projection
+xsize = 1496
+ysize = 2700
+xunits = "meter"
+yunits = "meter"
+xfirst = -640000
+xinc = 1000
+yfirst = -3355000
+yinc = 1000
+grid_mapping = crs
+grid_mapping_name = polar_stereographic
+straight_vertical_longitude_from_pole = -45.
+standard_parallel = 70.
+latitude_of_projection_origin = 90.
+false_easting = 0.
+false_northing = 0.
diff --git a/pism_terra/heatflux.py b/pism_terra/heatflux.py
new file mode 100644
index 0000000..c0f0106
--- /dev/null
+++ b/pism_terra/heatflux.py
@@ -0,0 +1,294 @@
+# Copyright (C) 2026 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+# mypy: disable-error-code="call-overload"
+# pylint: disable=unused-import,too-many-positional-arguments,broad-exception-caught
+
+
+"""
+Prepare Heatflow Maps.
+"""
+
+import logging
+import zipfile
+from pathlib import Path
+
+import pandas as pd
+import rioxarray # noqa: F401 (registers the .rio accessor)
+import xarray as xr
+from rasterio.warp import Resampling
+
+from pism_terra.download import download_archive, extract_archive
+from pism_terra.workflow import check_xr_lazy
+
+logger = logging.getLogger(__name__)
+
+# Lucazeau (2019), "Analysis and Mapping of an Updated Terrestrial Heat Flow
+# Data Set", G-Cubed, doi:10.1029/2019GC008389. The gridded "similarity"
+# prediction (``HFgrid14.csv``) ships in Supporting Information S01
+# (supplement file ``sup-0003``); the ``sup-0004``/S02 archive is the
+# *point* database (``NGHF.csv``), not the map.
+HF_URL = (
+ "https://agupubs.onlinelibrary.wiley.com/action/downloadSupplement"
+ "?doi=10.1029%2F2019GC008389&file=2019GC008389-sup-0003-Data_Set_SI-S01.zip"
+)
+HF_ARCHIVE_NAME = "2019GC008389-sup-0003-Data_Set_SI-S01.zip"
+HF_CSV_NAME = "HFgrid14.csv"
+
+
+def _find_hfgrid_csv(search_dir: Path) -> Path | None:
+ """
+ Locate the ``HFgrid14.csv`` grid file anywhere under *search_dir*.
+
+ Parameters
+ ----------
+ search_dir : pathlib.Path
+ Directory searched recursively for the Lucazeau grid CSV.
+
+ Returns
+ -------
+ pathlib.Path or None
+ Path to the first match, or ``None`` if no match is found.
+ """
+ matches = sorted(search_dir.rglob(HF_CSV_NAME))
+ return matches[0] if matches else None
+
+
+def prepare_heatflux_lucazeau(
+ output_path: Path | str,
+ extract_path: Path | str,
+ force_overwrite: bool = False,
+) -> Path:
+ """
+ Stage the Lucazeau (2019) global heat-flow map as a cloud-optimized Zarr.
+
+ Reads the gridded "similarity" heat-flow prediction (``HFgrid14.csv``),
+ reshapes the point list onto its native 0.5-degree latitude-longitude grid,
+ attaches CF metadata and an EPSG:4326 CRS, and writes a consolidated
+ (cloud-readable) Zarr store.
+
+ The source CSV is obtained in this order: an already-extracted
+ ``HFgrid14.csv`` under *extract_path*; any ``.zip`` placed under
+ *extract_path* (extracted and searched); otherwise an automatic download
+ from Wiley. The Wiley endpoint sits behind a Cloudflare JS challenge that
+ plain HTTP clients cannot pass, so for unattended runs download the archive
+ manually and drop it in *extract_path*.
+
+ Parameters
+ ----------
+ output_path : Path or str
+ Directory in which the ``heatflux_lucazeau.zarr`` store is written.
+ extract_path : Path or str
+ Working directory for the downloaded/placed archive and its extracted
+ CSV.
+ force_overwrite : bool, default False
+ If True, rebuild the Zarr even when it already exists (the source CSV is
+ reused when present; only re-downloaded if missing).
+
+ Returns
+ -------
+ pathlib.Path
+ Path to the written Zarr store.
+
+ Raises
+ ------
+ RuntimeError
+ If the automatic download does not yield a valid zip (e.g. blocked by
+ Cloudflare); the message explains the manual-download fallback.
+ FileNotFoundError
+ If ``HFgrid14.csv`` cannot be located after download/extraction.
+
+ Notes
+ -----
+ Heat-flow values are in milliwatts per square metre (``mW m-2``) and are
+ kept as published, including the small number of negative / very large
+ cells. PISM's ``bheatflx`` expects ``W m-2``; convert (``* 1e-3``) and clamp
+ downstream when consuming this field.
+ """
+ extract_path = Path(extract_path)
+ output_path = Path(output_path)
+ extract_path.mkdir(parents=True, exist_ok=True)
+ output_path.mkdir(parents=True, exist_ok=True)
+
+ zarr_store = output_path / "heatflux_lucazeau.zarr"
+ if zarr_store.exists() and not force_overwrite:
+ logger.info("Heat-flow Zarr already exists, skipping: %s", zarr_store)
+ return zarr_store
+
+ # 1) already-extracted CSV 2) a pre-placed zip 3) download from Wiley
+ csv_path = _find_hfgrid_csv(extract_path)
+ if csv_path is None:
+ for zip_path in sorted(extract_path.glob("*.zip")):
+ if zipfile.is_zipfile(zip_path):
+ logger.info("Extracting pre-placed archive %s", zip_path.name)
+ extract_archive(zip_path, extract_path / "lucazeau", force_overwrite=force_overwrite, verbose=False)
+ csv_path = _find_hfgrid_csv(extract_path)
+
+ if csv_path is None:
+ archive_dest = extract_path / HF_ARCHIVE_NAME
+ logger.info("Downloading Lucazeau heat-flow map from %s", HF_URL)
+ archive = download_archive(HF_URL, dest=archive_dest, force_overwrite=force_overwrite, verbose=True)
+ if not zipfile.is_zipfile(archive):
+ archive.unlink(missing_ok=True)
+ raise RuntimeError(
+ "Could not download the Lucazeau heat-flow archive automatically — the Wiley "
+ "endpoint is behind a Cloudflare challenge that HTTP clients cannot pass. "
+ f"Manually download\n {HF_URL}\n"
+ f"and place the .zip (or the extracted {HF_CSV_NAME}) in\n {extract_path}"
+ )
+ extract_archive(archive, extract_path / "lucazeau", force_overwrite=force_overwrite, verbose=True)
+ csv_path = _find_hfgrid_csv(extract_path)
+
+ if csv_path is None:
+ raise FileNotFoundError(f"{HF_CSV_NAME} not found under {extract_path} after download/extraction")
+
+ logger.info("Reading heat-flow grid from %s", csv_path)
+ # Semicolon-delimited; the published header misspells "longitude" as
+ # "longiyude" and leaves Hf_obs empty (NaN) for unobserved cells.
+ df = pd.read_csv(csv_path, sep=";").rename(columns={"longiyude": "longitude"})
+
+ # Point list -> regular lat/lon grid (the file is a complete 0.5-degree grid).
+ ds = (
+ df.set_index(["latitude", "longitude"])[["HF_pred", "sHF_pred", "Hf_obs"]]
+ .to_xarray()
+ .rename(
+ {
+ "latitude": "lat",
+ "longitude": "lon",
+ "HF_pred": "heatflux",
+ "sHF_pred": "heatflux_uncertainty",
+ "Hf_obs": "heatflux_observed",
+ }
+ )
+ .sortby("lat")
+ .sortby("lon")
+ )
+
+ ds["lat"].attrs.update(standard_name="latitude", long_name="latitude", units="degrees_north")
+ ds["lon"].attrs.update(standard_name="longitude", long_name="longitude", units="degrees_east")
+ ds["heatflux"].attrs.update(
+ standard_name="upward_geothermal_heat_flux_at_ground_level",
+ long_name="predicted surface heat flow (Lucazeau 2019 similarity method)",
+ units="mW m-2",
+ )
+ ds["heatflux_uncertainty"].attrs.update(
+ long_name="standard deviation of predicted surface heat flow", units="mW m-2"
+ )
+ ds["heatflux_observed"].attrs.update(long_name="observed surface heat flow where available", units="mW m-2")
+ ds.attrs.update(
+ title="Lucazeau (2019) global terrestrial heat-flow map (HFgrid14)",
+ references="Lucazeau, F. (2019), doi:10.1029/2019GC008389",
+ source=HF_URL,
+ )
+
+ ds = (
+ ds.rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=False)
+ .rio.write_crs("EPSG:4326", inplace=False)
+ .rio.write_grid_mapping("spatial_ref", inplace=False)
+ )
+
+ # Small global grid -> a single chunk; consolidated metadata makes the store
+ # cloud-readable in one metadata request.
+ ds = ds.chunk({"lat": ds.sizes["lat"], "lon": ds.sizes["lon"]})
+ logger.info("Writing cloud-optimized Zarr to %s", zarr_store)
+ ds.to_zarr(zarr_store, mode="w", consolidated=True)
+
+ return zarr_store
+
+
+def heatflux_from_grid(
+ target_grid: xr.Dataset,
+ dataset: str = "lucazeau",
+ path: Path | str = "tmp.nc",
+ bucket: str = "pism-cloud-data",
+ prefix: str = "glacier",
+ force_overwrite: bool = False,
+) -> xr.Dataset:
+ """
+ Interpolate a staged heat-flow map onto a target grid as PISM ``bheatflx``.
+
+ Opens the cloud-optimized Zarr produced by
+ :func:`prepare_heatflux_lucazeau` from
+ ``s3://{bucket}/{prefix}/heatflux/heatflux_{dataset}.zarr``, reprojects the
+ global field onto *target_grid*, converts to PISM's geothermal-flux units,
+ and writes the result to ``path``. A valid cache at ``path`` is reused unless
+ ``force_overwrite=True``.
+
+ Parameters
+ ----------
+ target_grid : xarray.Dataset
+ Grid (with a projected CRS in ``spatial_ref``) the heat flow is aligned to.
+ dataset : str, default ``"lucazeau"``
+ Heat-flow dataset name; selects ``heatflux_{dataset}.zarr`` on S3.
+ path : str or pathlib.Path, default ``"tmp.nc"``
+ Output/cache NetCDF (e.g. ``bheatflux_{rgi_id}.nc``).
+ bucket : str, default ``"pism-cloud-data"``
+ S3 bucket holding the staged heat-flow Zarr.
+ prefix : str, default ``"glacier"``
+ S3 key prefix under *bucket*.
+ force_overwrite : bool, default ``False``
+ If ``True``, ignore any cache at ``path`` and regenerate.
+
+ Returns
+ -------
+ xarray.Dataset
+ Single-variable dataset ``bheatflx`` (``W m-2``) on *target_grid*.
+
+ Notes
+ -----
+ The source map is in ``mW m-2``; it is converted to ``W m-2`` (``* 1e-3``)
+ and the handful of unphysical negative source cells are clamped to zero so
+ the field is a valid PISM geothermal-flux input.
+ """
+ print("")
+ print("Generate Geothermal Heat Flux")
+ print("-" * 120)
+
+ if check_xr_lazy(path) and not force_overwrite:
+ logger.info("Using cached heat-flow file %s", path)
+ return xr.open_dataset(path)
+
+ path = Path(path)
+ path.unlink(missing_ok=True)
+
+ uri = f"s3://{bucket}/{prefix}/heatflux/heatflux_{dataset}.zarr".replace("//", "/").replace("s3:/", "s3://")
+ logger.info("Opening heat-flow Zarr %s", uri)
+ storage_options = {"anon": True} if uri.startswith("s3://") else None
+ ds = xr.open_zarr(uri, consolidated=True, storage_options=storage_options, chunks={})
+
+ hf = ds["heatflux"].rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=False)
+ if hf.rio.crs is None:
+ hf = hf.rio.write_crs("EPSG:4326")
+
+ # Interpolate the global 0.5-degree field onto the projected target grid.
+ bheatflx = hf.rio.reproject_match(target_grid, resampling=Resampling.bilinear)
+
+ # mW/m^2 -> W/m^2 (PISM `bheatflx` units); clamp unphysical negative cells.
+ bheatflx = (bheatflx * 1.0e-3).clip(min=0.0)
+ bheatflx.name = "bheatflx"
+ bheatflx.attrs = {
+ "standard_name": "upward_geothermal_heat_flux_at_ground_level",
+ "long_name": "upward geothermal flux at the bottom bedrock surface",
+ "units": "W m-2",
+ "source": f"Lucazeau (2019) heat-flow map ({dataset})",
+ }
+ bheatflx = bheatflx.rio.write_crs(target_grid.rio.crs).rio.write_grid_mapping()
+
+ out = bheatflx.to_dataset()
+ out.to_netcdf(path, engine="h5netcdf")
+ logger.info("Heat flux saved to %s", path)
+ return out
diff --git a/pism_terra/ismip7/experiments.py b/pism_terra/ismip7/experiments.py
new file mode 100644
index 0000000..b97aa8b
--- /dev/null
+++ b/pism_terra/ismip7/experiments.py
@@ -0,0 +1,122 @@
+# Copyright (C) 2026 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+"""
+ISMIP7 Core Experiment definitions keyed by ``counter_id``.
+
+A single ISMIP7 ``counter_id`` (``C001`` … ``C008``) uniquely determines the
+experiment identity: the ISMIP7 ``experiment_id`` of the submitted product, the
+forcing ``pathway`` used to stage the climate/ocean inputs, the ESM (``gcm``),
+and the projection end year. Encoding that mapping here lets one config field
+(``run_info.counter``) drive both staging (``pism_terra/ismip7/greenland/stage.py``)
+and running (``pism_terra/ismip7/greenland/run.py``), removing the previous
+redundancy between ``run_info.experiment`` and ``[campaign].pathway`` / ``gcms``.
+
+Source: the "Core Experiment Overview" table in the ISMIP7 Protocol Overview
+spreadsheet (updated 2026-06-29), rows C001–C008.
+
+Notes on the two Historical entries (C001/C002): the protocol asks for the
+historical run only, but we still continue into a projection (ssp585 forcing,
+ending 2100). For those counters the **historical** leg is the ISMIP7 product and
+the projection leg is an internal continuation (flat filenames). For C003–C008 the
+**projection** leg is the ISMIP7 product and the historical spin-up leg is kept but
+written with flat filenames.
+
+C009–C011 (CTRL2015 control run and OCX observationally-constrained experiment)
+need distinct forcing/time handling and are intentionally omitted for now.
+
+This module is deliberately dependency-free (it must not import
+:mod:`pism_terra.config`) so it can be imported from the config resolver without
+creating an import cycle.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class CoreExperiment:
+ """
+ Resolved identity of one ISMIP7 Core experiment.
+
+ Attributes
+ ----------
+ experiment_id : str
+ ISMIP7 ``experiment_id`` of the submitted product leg, one of
+ ``"historical"``, ``"ssp126"``, ``"ssp370"``, ``"ssp585"``.
+ pathway : str
+ Forcing pathway used to build the staged input filenames
+ ``ismip7_greenland_{forcing}_{pathway}_{gcm}_{version}.nc``. For the
+ Historical counters this is ``"ssp585"`` (the pathway whose forcing the
+ internal continuation uses); the historical climate is embedded in that
+ merged forcing file.
+ esm_id : str
+ The Earth System Model (GCM), e.g. ``"CESM2-WACCM"`` or ``"MRI-ESM2-0"``.
+ proj_end_year : int
+ Calendar year the projection leg ends (``2100`` or ``2300``).
+ product_leg : str
+ Which of the two forward legs is the ISMIP7 submission product, either
+ ``"historical"`` or ``"projection"``. The other leg is written with flat
+ (non-ISMIP7) filenames.
+ """
+
+ experiment_id: str
+ pathway: str
+ esm_id: str
+ proj_end_year: int
+ product_leg: str
+
+
+# Core Experiment Overview (ISMIP7 Protocol Overview, updated 2026-06-29), C001–C008.
+CORE_EXPERIMENTS: dict[str, CoreExperiment] = {
+ "C001": CoreExperiment("historical", "ssp585", "CESM2-WACCM", 2100, "historical"),
+ "C002": CoreExperiment("historical", "ssp585", "MRI-ESM2-0", 2100, "historical"),
+ "C003": CoreExperiment("ssp370", "ssp370", "CESM2-WACCM", 2100, "projection"),
+ "C004": CoreExperiment("ssp370", "ssp370", "MRI-ESM2-0", 2100, "projection"),
+ "C005": CoreExperiment("ssp126", "ssp126", "CESM2-WACCM", 2300, "projection"),
+ "C006": CoreExperiment("ssp126", "ssp126", "MRI-ESM2-0", 2300, "projection"),
+ "C007": CoreExperiment("ssp585", "ssp585", "CESM2-WACCM", 2300, "projection"),
+ "C008": CoreExperiment("ssp585", "ssp585", "MRI-ESM2-0", 2300, "projection"),
+}
+
+
+def resolve_counter(counter: str) -> CoreExperiment:
+ """
+ Look up the :class:`CoreExperiment` for an ISMIP7 ``counter_id``.
+
+ Parameters
+ ----------
+ counter : str
+ ISMIP7 counter id, e.g. ``"C003"`` (case-insensitive).
+
+ Returns
+ -------
+ CoreExperiment
+ The resolved experiment definition.
+
+ Raises
+ ------
+ ValueError
+ If ``counter`` is not a supported Core experiment id.
+ """
+ key = str(counter).strip().upper()
+ try:
+ return CORE_EXPERIMENTS[key]
+ except KeyError as exc:
+ supported = ", ".join(sorted(CORE_EXPERIMENTS))
+ raise ValueError(f"unknown ISMIP7 counter {counter!r}; supported: {supported}") from exc
diff --git a/pism_terra/ismip7/greenland/forcing.py b/pism_terra/ismip7/greenland/forcing.py
index f5ec8f1..c98df30 100644
--- a/pism_terra/ismip7/greenland/forcing.py
+++ b/pism_terra/ismip7/greenland/forcing.py
@@ -16,22 +16,28 @@
# along with PISM; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
-# pylint: disable=too-many-positional-arguments,unused-import
+# pylint: disable=too-many-positional-arguments,unused-import,broad-exception-caught
"""
Prepare ISMIP7 Greenland data sets.
"""
+import logging
import os
import re
+import shutil
+import subprocess
import time
from argparse import ArgumentParser
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import as_completed as cf_as_completed
from pathlib import Path
-from typing import Any, Sequence
+from typing import Any, Literal, Sequence
import cf_xarray
import geopandas as gpd
import numpy as np
import pandas as pd
+import requests
import rioxarray # pylint: disable=unused-import
import toml
import xarray as xr
@@ -39,21 +45,262 @@
from cdo import Cdo
from dask.distributed import Client, as_completed
from pyfiglet import Figlet
+from rasterio.enums import Resampling
from tqdm.auto import tqdm
from pism_terra.domain import create_domain
from pism_terra.download import (
download_earthaccess,
+ download_file,
download_gebco,
download_netcdf,
file_localizer,
)
from pism_terra.raster import create_ds
from pism_terra.vector import dissolve
-from pism_terra.workflow import check_xr_fully, check_xr_lazy
+from pism_terra.workflow import (
+ check_xr_fully,
+ check_xr_lazy,
+ drop_geotransform_attr,
+ stamp_grid_mapping,
+)
xr.set_options(keep_attrs=True)
+logger = logging.getLogger(__name__)
+
+
+ISMIP7_GLOBUS_BASE = "https://g-ab4495.8c185.08cc.data.globus.org/ISMIP7/GrIS"
+
+
+def _make_url(year, ice_sheet, gcm, pathway, short_hand, m_var, version):
+ """
+ Build the Globus HTTPS URL for one ISMIP7 GrIS forcing file.
+
+ Mirrors the public Globus directory layout::
+
+ {base}/{gcm}/{pathway}/{short_hand}/{m_var}/{version}/.nc
+
+ where ```` is ``{m_var}_{ice_sheet}_{gcm}_{pathway}_{short_hand}_{version}_{year}.nc``
+ (without the ``short_hand`` segments when it equals ``"none"``).
+
+ Parameters
+ ----------
+ year : int
+ Year of the forcing file.
+ ice_sheet : str
+ Ice-sheet identifier embedded in the filename (e.g. ``"GrIS"`` or
+ ``"AIS"``).
+ gcm : str
+ GCM name (e.g. ``"CESM2-WACCM"``).
+ pathway : str
+ Emissions pathway (e.g. ``"historical"``, ``"ssp585"``).
+ short_hand : str
+ Short-hand identifier for the forcing type (e.g. ``"SDBN1-1000m"``)
+ or ``"none"`` when the variable lives in a per-GCM/pathway tree
+ without the short-hand segment.
+ m_var : str
+ Variable name (e.g. ``"acabf"``, ``"tas"``).
+ version : str
+ Version string (e.g. ``"v1"``).
+
+ Returns
+ -------
+ str
+ Globus HTTPS URL for the file.
+ """
+ fname = (
+ f"{m_var}_{ice_sheet}_{gcm}_{pathway}_{short_hand}_{version}_{year}.nc"
+ if short_hand != "none"
+ else f"{m_var}_{ice_sheet}_{gcm}_{pathway}_{version}_{year}.nc"
+ )
+ parts = [ISMIP7_GLOBUS_BASE, gcm, pathway]
+ if short_hand != "none":
+ parts.append(short_hand)
+ parts.extend([m_var, version, fname])
+ return "/".join(parts)
+
+
+class GlobusAuthRequired(RuntimeError):
+ """Globus refused the download because no valid bearer token was supplied."""
+
+
+def _globus_headers() -> dict[str, str]:
+ """
+ Build header.
+
+ Build the Authorization header for Globus HTTPS downloads, if a token
+ is available. Reads ``GLOBUS_ACCESS_TOKEN`` from the environment.
+
+ Returns
+ -------
+ dict
+ Authorization header.
+ """
+ token = os.environ.get("GLOBUS_ACCESS_TOKEN")
+ return {"Authorization": f"Bearer {token}"} if token else {}
+
+
+def _download_one(url: str, dest: Path, force_overwrite: bool = False, timeout: int = 600) -> Path:
+ """
+ Download a single ISMIP7 NetCDF from Globus to ``dest``.
+
+ Streams the response to a temporary ``.part`` file and renames on
+ completion so partial downloads never look like valid caches. If
+ ``dest`` already exists and ``force_overwrite`` is False, it is
+ returned unchanged.
+
+ If ``GLOBUS_ACCESS_TOKEN`` is set in the environment, it is sent as a
+ Bearer token in the ``Authorization`` header. If the server tries to
+ redirect to ``auth.globus.org`` (i.e. the collection requires login),
+ the function aborts with :class:`GlobusAuthRequired` rather than
+ chasing the auth flow.
+
+ Parameters
+ ----------
+ url : str
+ Source URL (typically built by :func:`_make_url`).
+ dest : Path
+ Local target path.
+ force_overwrite : bool, default False
+ Re-download even if ``dest`` is already present.
+ timeout : int, default 600
+ Per-request timeout in seconds.
+
+ Returns
+ -------
+ Path
+ ``dest`` (after a successful download or as a cache hit).
+
+ Raises
+ ------
+ GlobusAuthRequired
+ If the request gets redirected to Globus Auth (collection requires
+ authenticated access).
+ """
+ if dest.exists() and not force_overwrite:
+ return dest
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ tmp = dest.with_suffix(dest.suffix + ".part")
+ headers = _globus_headers()
+ with requests.get(url, headers=headers, stream=True, timeout=timeout, allow_redirects=False) as r:
+ # Follow data-server redirects (g-…data.globus.org → real node) but
+ # bail loudly on auth redirects so we don't hammer auth.globus.org.
+ while r.is_redirect:
+ location = r.headers.get("Location", "")
+ if "auth.globus.org" in location:
+ raise GlobusAuthRequired(
+ "Globus collection requires authentication. "
+ "Set GLOBUS_ACCESS_TOKEN with a valid Bearer token "
+ "(see notes in pism_terra.ismip7.greenland.forcing)."
+ )
+ r.close()
+ r = requests.get(location, headers=headers, stream=True, timeout=timeout, allow_redirects=False)
+ r.raise_for_status()
+ with open(tmp, "wb") as f:
+ for chunk in r.iter_content(chunk_size=1 << 20):
+ if chunk:
+ f.write(chunk)
+ tmp.rename(dest)
+ return dest
+
+
+def _download_many(
+ pairs: Sequence[tuple[str, Path]],
+ max_workers: int = 2,
+ desc: str = "Downloading ISMIP7",
+ force_overwrite: bool = False,
+) -> list[Path]:
+ """
+ Download a list of ``(url, dest)`` pairs in parallel.
+
+ Parameters
+ ----------
+ pairs : sequence of (str, pathlib.Path)
+ URLs to fetch and the local destinations to write them to.
+ max_workers : int, default 8
+ Number of concurrent download workers.
+ desc : str, default "Downloading ISMIP7"
+ Progress-bar description.
+ force_overwrite : bool, default False
+ Re-download even if a cached file already exists at the destination.
+
+ Returns
+ -------
+ list of Path
+ Paths to all successfully downloaded files (in completion order).
+ """
+ results: list[Path] = []
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ futures = {executor.submit(_download_one, url, dest, force_overwrite): (url, dest) for url, dest in pairs}
+ pbar = tqdm(cf_as_completed(futures), total=len(futures), desc=desc, unit="file")
+ for fut in pbar:
+ url, dest = futures[fut]
+ try:
+ results.append(fut.result())
+ pbar.set_postfix_str(f"{dest.name} ✓")
+ except GlobusAuthRequired as exc:
+ # Don't keep hammering: cancel remaining work and surface a clear error.
+ pbar.set_postfix_str("auth required — aborting")
+ for pending in futures:
+ pending.cancel()
+ raise exc
+ except Exception as exc:
+ pbar.set_postfix_str(f"{dest.name} ✗")
+ logger.error("Failed to download %s: %s", url, exc)
+ return results
+
+
+def _local_path(year, data_path, ice_sheet, gcm, pathway, short_hand, m_var, version):
+ """
+ Build the local file path for an ISMIP7 forcing variable and year.
+
+ Used when reading from a local mirror of the Globus tree. The on-disk
+ layout matches the Globus URL exactly — same subdir hierarchy and same
+ filename — so the only flexibility is whether *data_path* already includes
+ the trailing ``GrIS`` segment or sits above it.
+
+ Parameters
+ ----------
+ year : int
+ Year of the forcing file.
+ data_path : pathlib.Path
+ Root of the local mirror; expected to contain a ``/``
+ subdirectory that mirrors the Globus tree (e.g.
+ ``~/storstrommen/ISMIP7/`` for ``ice_sheet="GrIS"``).
+ ice_sheet : str
+ Ice-sheet subdirectory under *data_path* and segment of the local
+ filename (``"GrIS"`` or ``"AIS"``).
+ gcm : str
+ GCM name (e.g. ``"CESM2-WACCM"``).
+ pathway : str
+ Emissions pathway (e.g. ``"historical"``, ``"ssp585"``).
+ short_hand : str
+ Short-hand identifier for the forcing type, or ``"none"``.
+ m_var : str
+ Variable name (e.g. ``"acabf"``, ``"tas"``).
+ version : str
+ Version string (e.g. ``"v1"``).
+
+ Returns
+ -------
+ pathlib.Path
+ Path to the file under *data_path* (or under *data_path/GrIS* if that
+ is where the file actually lives). When neither candidate exists, the
+ first candidate is returned so callers can surface a sensible error.
+ """
+ fname = (
+ f"{m_var}_{ice_sheet}_{gcm}_{pathway}_{short_hand}_{version}_{year}.nc"
+ if short_hand != "none"
+ else f"{m_var}_{ice_sheet}_{gcm}_{pathway}_{version}_{year}.nc"
+ )
+ rel_parts = [gcm, pathway]
+ if short_hand != "none":
+ rel_parts.append(short_hand)
+ rel_parts.extend([m_var, version, fname])
+ rel = Path(*rel_parts)
+ return data_path / ice_sheet / rel
+
def _make_path(year, base_path, gcm, pathway, short_hand, m_var, version):
"""
@@ -95,7 +342,51 @@ def _make_path(year, base_path, gcm, pathway, short_hand, m_var, version):
return str(url)
+def _strip_fill_attrs(path: Path) -> None:
+ """
+ Strip ``_FillValue`` and ``missing_value`` attrs in place.
+
+ CDO and netCDF-4 both treat ``_FillValue`` as a reserved attribute and
+ refuse to delete it via the public API. NCO's ``ncatted`` is the only
+ reliable way to remove it as a pure metadata edit (no data rewrite).
+ All NaN / missing cells have already been filled by ``setmisstoc`` (and
+ the per-variable temperature fill in ``_merge_one_var``) by the time
+ this runs, so the attributes carry no remaining information.
+
+ Parameters
+ ----------
+ path : pathlib.Path
+ NetCDF file to edit in place.
+
+ Raises
+ ------
+ RuntimeError
+ If ``ncatted`` is not installed; install ``nco`` in the active
+ conda env to satisfy this dependency.
+ subprocess.CalledProcessError
+ If ``ncatted`` runs but fails (e.g. permissions, corrupt file).
+ """
+ if shutil.which("ncatted") is None:
+ raise RuntimeError("ncatted not found on PATH; install the 'nco' conda package")
+ # The "...,,d,," form means: attribute name ".._FillValue", any var
+ # (empty middle field), action "d" (delete), no value, no type.
+ subprocess.run(
+ [
+ "ncatted",
+ "-O",
+ "-h", # don't append a history line; the file stays bit-stable across reruns
+ "-a",
+ "_FillValue,,d,,",
+ "-a",
+ "missing_value,,d,,",
+ str(path),
+ ],
+ check=True,
+ )
+
+
def _process_single_forcing(
+ ice_sheet: Literal["AIS", "GrIS"],
gcm: str,
forcing: str,
base_path: Path,
@@ -109,12 +400,22 @@ def _process_single_forcing(
ismip7_to_pism: dict[str, str],
freq: str = "1mon",
calendar: str = "365_day",
-) -> Path:
+ data_path: Path | None = None,
+ staging_path: Path | None = None,
+) -> list[Path]:
"""
- Process a single GCM/forcing combination.
+ Process a single (GCM, pathway, forcing) combination into one output file.
+
+ Each pathway (historical, ssp???) is now its own task: no more coupling
+ the historical and projection epochs inside one worker. The output
+ filename embeds ``pathway`` and the ``start_year``/``end_year`` span,
+ e.g. ``ismip7_greenland_climate_historical_CESM2-WACCM_v2_1978_2014.nc``.
Parameters
----------
+ ice_sheet : {"AIS", "GrIS"}
+ Ice-sheet identifier; selects the subtree under the Globus base or
+ local mirror and is embedded in source filenames.
gcm : str
GCM name.
forcing : str
@@ -124,13 +425,14 @@ def _process_single_forcing(
output_path : Path
Output directory.
pathway : str
- Pathway name (e.g., "historical", "ssp585").
+ Pathway name (e.g., ``"historical"``, ``"ssp585"``).
version : str
- Version string (e.g., "v1").
+ Version string (e.g., ``"v1"``).
start_year : int
- Start year for time selection.
+ First year in the epoch (inclusive).
end_year : int
- End year for time selection.
+ Last year in the epoch (**inclusive** — ``end_year = 2300`` means
+ the year 2300 is processed).
short_hand : str
Short hand identifier for forcing.
fields : list[str]
@@ -141,43 +443,316 @@ def _process_single_forcing(
Frequency string for CDO time axis. Default is "1mon".
calendar : str, optional
Calendar type for CDO time axis. Default is "365_day".
+ data_path : pathlib.Path or None, optional
+ If given, read forcing files from this local mirror of the Globus
+ tree instead of downloading from ``base_path``. Files are looked up
+ with their Globus filename under ``data_path`` (or under
+ ``data_path/GrIS``). When ``None`` (default), the function downloads
+ from Globus and stores files under ``base_path``.
+ staging_path : pathlib.Path or None, optional
+ Directory for intermediate scratch (the per-variable cdo
+ ``mergetime`` tmp files). Auto-cleaned at the end of the function
+ via ``TemporaryDirectory``. When ``None`` (default),
+ ``output_path`` is used — but only the final merged file is left
+ in ``output_path`` either way.
Returns
-------
- Path
- Path to the output NetCDF file.
+ list[Path]
+ Paths to the produced NetCDF files (one per group: ``climate`` and
+ optionally ``climate_gradient`` when ``forcing == "climate"``).
"""
os.environ["HDF5_LOG_LEVEL"] = "0"
cdo = Cdo()
cdo.debug = True
- merge_cmds = []
- for m_var in fields:
- urls = [
- _make_path(year, base_path, gcm, pathway, short_hand, m_var, version)
- for year in range(start_year, end_year)
- ]
+ grid_file = file_localizer("s3://pism-cloud-data/ismip7_extra/grid.txt", dest=output_path)
+ tas_replace = ""
+
+ output_files = []
+
+ def _resolve(year: int, pathway_name: str, m_var: str) -> Path:
+ """
+ Return the local file path for one (year, pathway, var).
+
+ Parameters
+ ----------
+ year : int
+ Calendar year of the requested forcing slice.
+ pathway_name : str
+ Emissions pathway segment of the path (e.g. ``"historical"``).
+ m_var : str
+ ISMIP7 variable name (e.g. ``"acabf"``).
+
+ Returns
+ -------
+ pathlib.Path
+ Path under ``data_path`` (Globus-mirror filename) when
+ ``data_path`` was supplied, else the legacy ``_make_path``
+ location under ``base_path``.
+ """
+ if data_path is not None:
+ return _local_path(year, data_path, ice_sheet, gcm, pathway_name, short_hand, m_var, version)
+ # _make_path doesn't take ice_sheet (the segment isn't part of the legacy
+ # base_path layout).
+ return Path(_make_path(year, base_path, gcm, pathway_name, short_hand, m_var, version))
+
+ if data_path is None:
+ # Build (url, local_path) pairs for every (variable, year) we need
+ # for this pathway. ``end_year`` is inclusive per the campaign
+ # config convention, so the range hits ``end_year`` itself.
+ download_pairs: list[tuple[str, Path]] = []
+ for m_var in fields:
+ for year in range(start_year, end_year + 1):
+ url = _make_url(year, ice_sheet, gcm, pathway, short_hand, m_var, version)
+ download_pairs.append((url, _resolve(year, pathway, m_var)))
+
+ _download_many(download_pairs, desc=f"Download {gcm}/{pathway}/{forcing}")
+ else:
+ logger.info("Using local ISMIP7 forcing under %s for %s/%s/%s", data_path, gcm, pathway, forcing)
+
+ # cdo merges run on the resolved local paths (downloaded or pre-existing).
+ # Doing the per-variable mergetime in-process (alongside the final
+ # cross-variable merge) produced one shell invocation listing every
+ # (variable, year) source file. With ~6 variables × ~300 years that
+ # easily exceeds ARG_MAX. Split the work: per-variable mergetime/chname
+ # writes a tmp file; the final per-epoch cdo only sees one tmp per var.
+ import tempfile # pylint: disable=import-outside-toplevel
+
+ def _merge_one_var(
+ tmp_root: Path, epoch_label: str, pathway_name: str, start_year: int, end_year: int, m_var: str
+ ) -> Path:
+ """
+ Mergetime + chname for one (epoch, variable) into a tmp NetCDF.
+
+ Splitting the per-variable merge off the final cross-variable merge
+ keeps any one ``cdo`` invocation well under the ARG_MAX limit, even
+ for projection epochs that span hundreds of years.
+
+ Parameters
+ ----------
+ tmp_root : pathlib.Path
+ Directory for the per-variable tmp output.
+ epoch_label : str
+ Short label embedded in the tmp filename (e.g. ``"hist"`` or
+ ``"proj"``) to keep epochs distinct in the same tmp dir.
+ pathway_name : str
+ Emissions pathway segment of the source path (e.g.
+ ``"historical"`` or ``"ssp585"``).
+ start_year, end_year : int
+ Inclusive/exclusive year range for the source files.
+ m_var : str
+ ISMIP7 variable name (e.g. ``"acabf"``).
+
+ Returns
+ -------
+ pathlib.Path
+ Path to the per-variable tmp NetCDF.
+ """
+ paths = [_resolve(year, pathway_name, m_var) for year in range(start_year, end_year + 1)]
k, v = m_var, ismip7_to_pism[m_var]
- tas_replace = "-setrtoc,0,230,230 -setrtoc,303,403,303" if m_var == "tas" else ""
- merge_cmd = f"-chname,{k},{v} {tas_replace} -mergetime [ " + " ".join(str(f) for f in urls) + " ]"
- merge_cmds.append(merge_cmd)
- output_file = output_path / Path(f"ismip7_greenland_{forcing}_{pathway}_{gcm}_{version}_{start_year}_{end_year}.nc")
+ out = tmp_root / f"{epoch_label}_{m_var}.nc"
+ # Per-variable fill applied in the per-tmp stage so the outer
+ # ``setmisstoc,0`` has nothing to do for this variable. The outer
+ # fill is correct for SMB / runoff (no-ice cells == 0 mass change)
+ # but unphysical for surface temperature, which the source masks
+ # out as NaN over non-ice cells and (in some files) ships with
+ # literal 0 K values. Replace both with 260 K — well below the
+ # outer fill range, so any real cold temperature is preserved.
+ if m_var in ("ts", "tas"):
+ fill_op = " -setrtoc,-1,1,260 -setmisstoc,260"
+ elif m_var == "tf":
+ # Ocean thermal forcing: extrapolate valid ocean values into the
+ # masked gaps (nearest-neighbour) instead of letting the outer
+ # ``setmisstoc,0`` zero-fill them. A zero thermal forcing (ocean at
+ # the freezing point) breaks PICO's box-1 quadratic (T_star ~ 0 ->
+ # negative sqrt); real thermal forcing can dip slightly below 0, so
+ # do NOT treat near-zero values as fill here.
+ fill_op = " -setmisstonn"
+ elif m_var == "so":
+ # Ocean salinity: convert the near-zero land/fill values to missing,
+ # then extrapolate valid ocean salinity into the gaps
+ # (nearest-neighbour) rather than filling with a constant. Real ocean
+ # salinity is ~30-35 g/kg, never near zero, so [-1, 1] is safe to drop.
+ fill_op = " -setmisstonn -setrtomiss,-1,1"
+ else:
+ fill_op = ""
+ mergetime_chain = (
+ f"{tas_replace}{fill_op} -setgrid,{str(grid_file)} -mergetime [ " + " ".join(str(p) for p in paths) + " ]"
+ )
+ if m_var == "so":
+ # CMIP6 sea-water salinity ships with ``units = "psu"`` (practical
+ # salinity unit). PISM's ``ocean.th`` requires the numerically
+ # identical but udunits-parsable ``g/kg`` and refuses to read the
+ # file otherwise. Patch the attribute on the renamed variable in
+ # the same cdo invocation so we don't pay an extra read/write.
+ cdo.setattribute(
+ f"{v}@units=g/kg",
+ input=f"-chname,{k},{v} {mergetime_chain}",
+ output=str(out.resolve()),
+ options="-f nc4 -z zip_2",
+ )
+ else:
+ cdo.chname(
+ f"{k},{v}",
+ input=mergetime_chain,
+ output=str(out.resolve()),
+ options="-f nc4 -z zip_2",
+ )
+ return out
+
+ # ISMIP7 publishes "climate" fields at two cadences: ``acabf`` / ``mrro``
+ # / ``ts`` are monthly, while the elevation-gradient fields ``dacabfdz``
+ # and ``dmrrodz`` are annual. ``cdo -merge`` requires aligned time axes,
+ # so mixing the two would silently truncate the monthly streams to the 5
+ # annual timesteps (warning ``Input stream 2 has 5 timesteps. Stream 1
+ # has more timesteps, skipped!``). Emit two separate output files
+ # instead: the standard ``..._climate_...`` for the monthly fields and
+ # ``..._climate_gradient_...`` for the annual ones. Other forcings
+ # (ocean, …) keep the original single-group behavior.
+ annual_fields_set = {"dacabfdz", "dmrrodz", "dtsdz"}
+ if forcing == "climate":
+ monthly_fields = [f for f in fields if f not in annual_fields_set]
+ annual_fields = [f for f in fields if f in annual_fields_set]
+ groups: list[tuple[str, list[str], str, str]] = []
+ if monthly_fields:
+ groups.append(("climate", monthly_fields, freq, "01-16 12:00"))
+ if annual_fields:
+ # Mid-year anchor for the annual gradient timestamps. CDO's
+ # ``settbounds`` only accepts hour/day/month frequency tokens,
+ # so use ``12month`` (which both ``settbounds`` and ``settaxis``
+ # parse as one calendar year) instead of ``1yr``.
+ groups.append(("climate_gradient", annual_fields, "12month", "07-02 12:00"))
+ else:
+ groups = [(forcing, fields, freq, "01-16 12:00")]
+
+ # Intermediates (cdo ``mergetime`` tmps) live under ``staging_path``
+ # instead of ``output_path``. The tempdir is removed on ``with`` exit,
+ # so disk usage drops back to just the single output file per group.
+ staging_root = Path(staging_path) if staging_path is not None else output_path
+ staging_root.mkdir(parents=True, exist_ok=True)
+ with tempfile.TemporaryDirectory(prefix=f"_ismip7_{gcm}_{pathway}_{forcing}_", dir=str(staging_root)) as _tmp:
+ tmp_root = Path(_tmp)
+
+ for label, sub_fields, sub_freq, time_anchor in groups:
+ tmps = [
+ _merge_one_var(tmp_root, f"{pathway}_{label}", pathway, start_year, end_year, m_var)
+ for m_var in sub_fields
+ ]
+
+ # Output name embeds the epoch: the historical file and each
+ # projection file live side-by-side and are consumed
+ # independently by PISM's two-invocation forward driver
+ # (see pism_terra.ismip7.greenland.run._render_forward_run).
+ out_file = output_path / Path(
+ f"ismip7_greenland_{label}_{pathway}_{gcm}_{version}_{start_year}_{end_year}.nc"
+ )
+ # ``-merge`` is variadic; ``[ ... ]`` delimits the file list
+ # so python-cdo doesn't misinterpret the ``output=`` argument
+ # as another input stream.
+ cdo.setmisstoc(
+ 0,
+ input=(
+ f"-setgrid,{str(grid_file)} -settbounds,{sub_freq} "
+ f"-setreftime,1850-01-01 -settunits,hours -setcalendar,{calendar} "
+ f"-settaxis,'{start_year}-{time_anchor},,{sub_freq}' -merge [ "
+ + " ".join(str(p.resolve()) for p in tmps)
+ + " ]"
+ ),
+ output=str(out_file.resolve()),
+ options="-f nc4 -z zip_2",
+ )
+ _strip_fill_attrs(out_file)
+ output_files.append(out_file)
+
+ return output_files
- grid_file = file_localizer("s3://pism-cloud-data/ismip7_extra/grid.txt", dest=output_path)
- cdo.setmisstoc(
- 0,
- input=f"""-setgrid,{str(grid_file)} -settbounds,{freq} -setreftime,1850-01-01 -settunits,hours -setcalendar,{calendar} -settaxis,'{start_year}-01-16 12:00,,{freq}' -merge """
- + " ".join(merge_cmds),
- returnXDataset=False,
- options="-f nc4 -z zip_2",
- output=str(output_file.resolve()),
- )
- return output_file
+# Packaged GrIS basin polygons (attribute ``basin`` in 1..7) used for the mask.
+_BASIN_GPKG = Path(__file__).resolve().parents[2] / "data" / "gris-basins-w-shelves.gpkg"
+
+
+def basin_mask(target_grid: xr.Dataset | xr.DataArray) -> xr.DataArray:
+ """
+ Rasterize the GrIS basins onto ``target_grid`` as an integer mask.
+
+ Reads the packaged basin polygons (``gris-basins-w-shelves.gpkg``, integer
+ attribute ``basin`` in ``1..7``) and assigns each grid cell the ``basin`` value
+ of the polygon that covers it; cells outside every basin are ``0``.
+
+ Parameters
+ ----------
+ target_grid : xarray.Dataset or xarray.DataArray
+ Grid to rasterize onto. Must carry ``x``/``y`` coordinates in EPSG:3413.
+
+ Returns
+ -------
+ xarray.DataArray
+ ``int8`` basin mask named ``"basins"`` with values ``0..7``.
+ """
+ basins_gdf = gpd.read_file(_BASIN_GPKG).to_crs("EPSG:3413")
+
+ # Zero reference field on the target grid, with x/y + CRS so rio.clip works.
+ ref = xr.DataArray(
+ np.zeros((target_grid["y"].size, target_grid["x"].size), dtype="float32"),
+ dims=("y", "x"),
+ coords={"y": target_grid["y"], "x": target_grid["x"]},
+ ).rio.write_crs("EPSG:3413")
+
+ basins = xr.zeros_like(ref, dtype="int8")
+ for _, row in basins_gdf.iterrows():
+ try:
+ # Inside-polygon mask (clip keeps interior, sets exterior -> NaN).
+ inside = ref.rio.clip([row.geometry], drop=False, all_touched=True).notnull()
+ except Exception: # pylint: disable=broad-exception-caught # basin outside the grid
+ continue
+ basins = basins.where(~inside, int(row["basin"]))
+ basins = basins.astype("int8")
+ basins.name = "basins"
+ basins.attrs = {"units": "1", "long_name": "GrIS basin id (1-7, 0=outside)"}
+ return basins
+
+
+def add_basins_to_ocean_files(ocean_files: list[Path]) -> None:
+ """
+ Add the GrIS basin mask to each ocean forcing file, in place.
+
+ Opens each ocean file, rasterizes the packaged basin polygons onto that file's
+ grid via :func:`basin_mask`, and writes the result back as an ``int8``
+ ``basins`` variable. Files are rewritten atomically via a sibling ``*.tmp``
+ file. A file whose grid cannot be matched is logged and skipped rather than
+ aborting the whole batch.
+
+ Parameters
+ ----------
+ ocean_files : list of pathlib.Path
+ Ocean forcing NetCDFs to annotate.
+ """
+ for ocean_file in ocean_files:
+ try:
+ with xr.open_dataset(ocean_file) as ds:
+ ds = ds.load()
+ basins = basin_mask(ds)
+ ds["basins"] = basins
+ # Georeference basins like the file's other data variables (match
+ # their ``grid_mapping``, e.g. ``crs``, rather than the source field's).
+ grid_mapping = next(
+ (ds[v].attrs["grid_mapping"] for v in ds.data_vars if v != "basins" and "grid_mapping" in ds[v].attrs),
+ None,
+ )
+ if grid_mapping:
+ ds["basins"].attrs["grid_mapping"] = grid_mapping
+ encoding = {"basins": {"zlib": True, "complevel": 2, "_FillValue": None}}
+ tmp = ocean_file.with_name(ocean_file.name + ".tmp")
+ ds.to_netcdf(tmp, encoding=encoding, engine="h5netcdf")
+ os.replace(tmp, ocean_file)
+ logger.info("Added basin mask to %s", ocean_file.name)
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ logger.warning("Could not add basin mask to %s: %s", ocean_file, exc)
def prepare_observations(
- url: str,
+ url: Path | str,
input_path: Path | str,
output_path: Path | str,
config: dict,
@@ -216,44 +791,112 @@ def prepare_observations(
Returns
-------
dict[str, Path or str]
- Dictionary with keys ``"boot_file"`` and ``"heatflux_file"``
- mapping to their respective output paths.
+ Dictionary with keys ``"boot_file"``, ``"heatflux_file"``, and
+ ``"obs_file"`` mapping to their respective output paths. The
+ ``"obs_file"`` is an inverse-distance-weighted collapse of
+ ``vx_timeseries``/``vy_timeseries``, post-processed in the same
+ shape as :func:`pism_terra.glacier.observations.glacier_velocities_from_grid`.
"""
- name = url.split("/")[-1]
- obs_file = Path(input_path) / Path(name)
- if (not check_xr_lazy(obs_file)) or force_overwrite:
- ds_bm = download_netcdf(url)
+ rho_ice = 910.0 # constants.ice.density
+ rho_sea_water = 1028.0 # constants.sea_water.density
+ ice_free_thickness = 0.01 # geometry.ice_free_thickness_standard
+ sea_level = 0.0
+ alpha = 1.0 - rho_ice / rho_sea_water
+
+ # ``url`` may be a Path (when reading from a local mirror) or a string
+ # (when downloading from Globus). Normalize to str for split/download.
+ url_str = str(url)
+ name = url_str.rsplit("/", maxsplit=1)[-1]
+ boot_file = Path(input_path) / Path(name)
+ if (not check_xr_lazy(boot_file)) or force_overwrite:
+ ds_bm = download_netcdf(url_str)
else:
- ds_bm = xr.open_dataset(obs_file)
+ ds_bm = xr.open_dataset(boot_file)
+
+ ds_bm = ds_bm.rename_vars({"surface_grimp": "surface"})
+ ds_bm["surface"].attrs.update(
+ {"standard_name": "surface_altitude", "long_name": "ice surface elevation", "units": "m"}
+ )
if target_grid is not None:
- ds_bm_regridded = ds_bm[["bed", "thickness"]].regrid.conservative(target_grid)
+ ds_bm_regridded = ds_bm[["bed", "thickness", "surface", "mask"]].regrid.conservative(target_grid)
gebco_p = download_gebco(target_dir=input_path)
gebco = xr.open_dataset(gebco_p, chunks="auto").rio.write_crs("EPSG:4326")
- gebco_bm_regridded = gebco.rio.reproject_match(ds_bm_regridded.rio.write_crs("EPSG:3413")).compute()
- ds_bm_regridded["bed"] = ds_bm_regridded["bed"].where(
- ds_bm_regridded["bed"].notnull(), gebco_bm_regridded["elevation"]
- )
+ gebco_bm_regridded = gebco.rio.reproject_match(
+ ds_bm_regridded.rio.write_crs("EPSG:3413"), resampling=Resampling.bilinear
+ ).compute()
+ # Use GEBCO bathymetry over ocean (mask == 0), and also to fill any gaps in
+ # BedMachine's bed; keep BedMachine's bed everywhere else.
+ use_gebco = (ds_bm_regridded["mask"] == 0) | ds_bm_regridded["bed"].isnull()
+ ds_bm_regridded["bed"] = ds_bm_regridded["bed"].where(~use_gebco, gebco_bm_regridded["elevation"])
ds_bm_regridded = ds_bm_regridded.fillna(0)
else:
ds_bm_regridded = ds_bm
ftt_mask = xr.where(ds_bm_regridded["thickness"] > 0, 1, 0)
ftt_mask.name = "ftt_mask"
+
+ liafr = xr.where(ds_bm_regridded["mask"] == 0, 0, 1)
+ liafr.name = "land_ice_area_fraction_retreat"
+ liafr.attrs.update({"units": "1"})
+ liafr = liafr.astype("bool")
+
if surface_dem is not None:
+ surface_file = Path(input_path) / Path("surface_dem.nc")
bed = ds_bm_regridded["bed"]
- surface = download_netcdf(surface_dem)["surface"].regrid.conservative(target_grid)
+ if (not check_xr_lazy(surface_file)) or force_overwrite:
+ ds = download_netcdf(surface_dem)
+ ds.to_netcdf(surface_file)
+ else:
+ ds = xr.open_dataset(surface_file)
+ surface = ds["surface"].regrid.conservative(target_grid)
+ surface = surface.where(surface > 0, 0)
+ surface.name = "surface"
thickness = xr.where(surface > 0, surface - bed, 0)
+ # In the ocean (mask == 0) and on ice shelves (mask == 3) a positive
+ # surface is floating freeboard, not bed-referenced elevation, so recover
+ # the thickness from flotation: freeboard = alpha * H => H = surface / alpha.
+ mask = ds_bm_regridded["mask"]
+ thickness_from_flotation = surface / alpha
+ # Only recover floating thickness inside the GrIS basin polygons
+ # (basin > 0); outside them, mask 0/3 cells are spurious ocean/shelf
+ # that would otherwise pick up bogus thicknesses.
+ in_basin = basin_mask(target_grid) > 0
+ is_floating = (surface > 0) & ((mask == 0) | (mask == 3)) & in_basin
+ thickness = xr.where(is_floating, thickness_from_flotation, thickness)
thickness = thickness.where(thickness > 10, 0)
thickness.name = "thickness"
thickness.attrs.update(ds_bm_regridded["thickness"].attrs)
- boot = xr.merge([bed, ftt_mask, surface, thickness])
+ boot = xr.merge([bed, ftt_mask, surface, thickness, liafr])
else:
- boot = xr.merge([ds_bm_regridded[["bed", "thickness"]], ftt_mask])
- boot = boot.fillna(0)
- ds = xr.merge([boot, ds_bm["mapping"]])
+ boot = xr.merge([ds_bm_regridded[["bed", "thickness", "surface"]], ftt_mask, liafr])
+
+ # Ellesmere Island sits inside the domain but is not part of the modeled
+ # Greenland ice sheet; force it to deep ocean so no ice can grow there.
+ # Inside the polygon set bed = -2000 m, surface = 0 m, thickness = 0 m.
+ ellesmere_gpkg = Path(__file__).resolve().parents[2] / "data" / "ellsmere.gpkg"
+ ellesmere = gpd.read_file(ellesmere_gpkg).to_crs("EPSG:3413")
+ if len(ellesmere) == 0:
+ logger.warning("%s has no geometry; skipping Ellesmere override", ellesmere_gpkg.name)
+ else:
+ try:
+ # Inside-polygon mask on the boot grid: clip a constant field (inside
+ # kept, outside -> NaN), so notnull() marks the polygon interior.
+ inside = (
+ xr.ones_like(boot["bed"])
+ .rio.write_crs("EPSG:3413")
+ .rio.clip(ellesmere.geometry, drop=False, all_touched=True)
+ .notnull()
+ )
+ boot["bed"] = boot["bed"].where(~inside, -2000.0)
+ boot["surface"] = boot["surface"].where(~inside, 0.0)
+ boot["thickness"] = boot["thickness"].where(~inside, 0.0)
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ logger.warning("Ellesmere override skipped (%s)", exc)
+ boot = boot.fillna(0)
+ ds = boot
geo = (
ds_bm[["geothermal_heat_flux1"]]
.rename_dims({"x1km": "x", "y1km": "y"})
@@ -261,7 +904,9 @@ def prepare_observations(
.regrid.conservative(target_grid)
)
geo = geo.where(geo != -9999, 0.042)
+ geo = geo.where(geo <= 1e36, 0.042)
+ ds["surface"].attrs.update({"standard_name": "surface_altitude", "units": "m"})
ds["bed"].attrs.update({"standard_name": "bedrock_altitude", "units": "m"})
ds = ds.rename_vars({k: v for k, v in config["ismip7_to_pism"].items() if k in ds}).drop_vars(
["crs", "spatial_ref"], errors="ignore"
@@ -269,26 +914,138 @@ def prepare_observations(
for v in ds.data_vars:
ds[v].attrs.pop("coordinates", None)
ds[v].encoding.pop("coordinates", None)
+ # BedMachine variables ship with stale per-variable ``grid_mapping``
+ # attrs (e.g. ``surface:grid_mapping = "polar_stereographic"``) that
+ # point at variables we never write. ``stamp_grid_mapping`` would
+ # canonicalize them on its own, but stripping here keeps the
+ # intermediate state cleaner and avoids surprises if the helper's
+ # heuristics change.
+ ds[v].attrs.pop("grid_mapping", None)
+
+ # ``write_crs`` returns a new dataset — the old code threw the return
+ # away, leaving ``ds`` without any ``mapping`` variable and with the
+ # stale BedMachine attrs as the only grid-mapping evidence.
+ ds = ds.rio.write_crs("EPSG:3413", grid_mapping_name="mapping").rio.write_coordinate_system()
+ drop_geotransform_attr(ds)
+ ds = stamp_grid_mapping(ds, name="mapping")
- resolution = int(ds.x[1] - ds.x[0])
- obs_file = output_path / Path(f"boot_g{resolution}m_GreenlandObsISMIP7-v1.3.nc")
comp = {"zlib": True, "complevel": 2}
- encoding = {var: comp for var in ds.data_vars}
- encoding.update({var: {"_FillValue": None} for var in list(ds.data_vars) + list(ds.coords)})
- ds.to_netcdf(obs_file, encoding=encoding)
+ for var in list(ds.data_vars) + list(ds.coords):
+ ds[var].encoding.update({"_FillValue": None})
+ for var in list(ds.data_vars):
+ ds[var].encoding.update(comp)
+
+ resolution = int(ds.x[1] - ds.x[0])
+ boot_file = output_path / Path(f"boot_g{resolution}m_GreenlandObsISMIP7-v1.3.nc")
+
+ ds.to_netcdf(boot_file, engine="h5netcdf")
geo["bheatflx"].attrs.pop("coordinates", None)
geo["bheatflx"].encoding.pop("coordinates", None)
geo = geo.drop_vars("spatial_ref", errors="ignore")
- geo["mapping"] = ds_bm["mapping"]
for v in geo.data_vars:
geo[v].attrs.pop("coordinates", None)
geo[v].encoding.pop("coordinates", None)
+ geo[v].attrs.pop("grid_mapping", None)
+ geo = geo.rio.write_crs("EPSG:3413", grid_mapping_name="mapping").rio.write_coordinate_system()
+ drop_geotransform_attr(geo)
+ geo = stamp_grid_mapping(geo, name="mapping")
geo_file = output_path / Path(f"heatflux_g{resolution}m_GreenlandObsISMIP7-v1.3.nc")
geo_encoding = {var: {"_FillValue": None} for var in list(geo.data_vars) + list(geo.coords)}
- geo.to_netcdf(geo_file, encoding=geo_encoding)
+ for var in geo.data_vars:
+ # See the obs write below: a per-variable encoding dict replaces the
+ # variable's ``.encoding``, so preserve the CF ``grid_mapping`` key.
+ grid_mapping = geo[var].encoding.get("grid_mapping")
+ if grid_mapping:
+ geo_encoding[var]["grid_mapping"] = grid_mapping
+ geo.to_netcdf(geo_file, encoding=geo_encoding, engine="h5netcdf")
+
+ # Velocity observations: collapse the ISMIP7 vx/vy time series with the
+ # inverse-distance-weighting recipe used in pism-ragis
+ # (data/05_prepare_itslive.py), then mirror the post-processing in
+ # pism_terra.glacier.observations.glacier_velocities_from_grid — fillna
+ # for u/v_observed and emit zeta_fixed_mask / vel_misfit_weight so the
+ # downstream PISM inverse run knows which cells carry trustable obs.
+
+ ice_mask = ds_bm["icemask_promice"]
+ vel = ds_bm[["vx_mosaic", "vy_mosaic"]].rename_vars({"vx_mosaic": "vx", "vy_mosaic": "vy"})
- return {"boot_file": obs_file, "heatflux_file": geo_file}
+ if target_grid is not None:
+ vel = vel.regrid.conservative(target_grid)
+ ice_mask = ice_mask.regrid.conservative(target_grid)
+
+ ice_mask = ice_mask > 0.5
+
+ # Integer GrIS basin mask (1..7, 0 outside) rasterized from the packaged
+ # basin polygons onto the target grid.
+ basins = basin_mask(target_grid)
+
+ # Grounded ice via PISM's flotation criterion (src/util/Mask.hh): ice is
+ # grounded where its base rests on the bed (not floating) and ice is present.
+ # hgrounded = bed + thickness; hfloating = sea_level + alpha * thickness
+ # alpha = 1 - rho_ice / rho_sea_water; floating if hfloating > hgrounded;
+ # ice_free if thickness <= ice_free_thickness_standard.
+ # Uses the (regridded) boot geometry, which is on the same grid as ``vel``.
+ bed = boot["bed"]
+ thk = boot["thickness"]
+ grounded_ice = (bed + thk >= sea_level + alpha * thk) & (thk > ice_free_thickness) & ice_mask
+
+ vel["v"] = ((vel["vx"].fillna(0) ** 2 + vel["vy"].fillna(0) ** 2) ** 0.5).astype("float32")
+ vel["u_observed"] = vel["vx"].fillna(0).astype("float32")
+ vel["v_observed"] = vel["vy"].fillna(0).astype("float32")
+ # zeta is FREE (0) where there is grounded ice and FIXED (1) elsewhere;
+ # the misfit weight is the inverse (1 = trust obs on grounded ice, 0 = ignore).
+ vel["zeta_fixed_mask"] = xr.where(grounded_ice, 0, 1).astype("int8")
+ vel["zeta_fixed_mask"].attrs.update({"units": "1", "long_name": "tauc_unchanging integer mask (1=fixed)"})
+ vel["vel_misfit_weight"] = xr.where(grounded_ice, 1, 0).astype("int8")
+ vel["vel_misfit_weight"].attrs.update({"units": "1", "long_name": "misfit weight (1=trust obs, 0=ignore)"})
+ vel["basins"] = basins
+
+ vel = vel.rio.write_crs("EPSG:3413", grid_mapping_name="mapping").rio.write_coordinate_system()
+ vel["x"].attrs.update(
+ {
+ "standard_name": "projection_x_coordinate",
+ "long_name": "x coordinate of projection",
+ "units": "m",
+ "axis": "X",
+ }
+ )
+ vel["y"].attrs.update(
+ {
+ "standard_name": "projection_y_coordinate",
+ "long_name": "y coordinate of projection",
+ "units": "m",
+ "axis": "Y",
+ }
+ )
+ vel["x"].encoding["_FillValue"] = None
+ vel["y"].encoding["_FillValue"] = None
+ for v in vel.data_vars:
+ vel[v].attrs.pop("coordinates", None)
+ vel[v].encoding.pop("coordinates", None)
+ vel[v].attrs.pop("grid_mapping", None)
+ for k in ("scale_factor", "add_offset", "AREA_OR_POINT"):
+ vel[v].attrs.pop(k, None)
+ vel[v].encoding.pop(k, None)
+ drop_geotransform_attr(vel)
+ vel = stamp_grid_mapping(vel, name="mapping")
+ vel = vel.drop_vars(["crs", "spatial_ref"], errors="ignore")
+
+ obs_file = output_path / Path(f"obs_g{resolution}m_GreenlandObsISMIP7-v1.3.nc")
+ vel_encoding: dict[str, dict[str, Any]] = {
+ var: {"_FillValue": None} for var in list(vel.data_vars) + list(vel.coords)
+ }
+ for var in vel.data_vars:
+ vel_encoding[var].update(comp)
+ # A per-variable encoding dict passed to ``to_netcdf`` replaces the
+ # variable's ``.encoding``, so the CF ``grid_mapping`` key set by
+ # stamp_grid_mapping would be dropped. Carry it through explicitly.
+ grid_mapping = vel[var].encoding.get("grid_mapping")
+ if grid_mapping:
+ vel_encoding[var]["grid_mapping"] = grid_mapping
+ vel.to_netcdf(obs_file, encoding=vel_encoding, engine="h5netcdf")
+
+ return {"boot_file": boot_file, "heatflux_file": geo_file, "obs_file": obs_file}
def prepare_calfin(
@@ -342,7 +1099,7 @@ def prepare_calfin(
if (not check_xr_lazy(p_fn)) or force_overwrite:
- tmp_path = output_path.parent / Path(output_path.name + "_tmp")
+ tmp_path = output_path.parent / Path("calfin")
# Download CALFIN data
retreat_files = download_earthaccess(
@@ -372,7 +1129,7 @@ def prepare_calfin(
groups = [(date, df) for date, df in calfin.groupby(pd.Grouper(freq=freq)) if len(df) > 0]
with Client(n_workers=n_workers, threads_per_worker=1) as client:
- print(f"Dask dashboard: {client.dashboard_link}")
+ logger.info("Dask dashboard: %s", client.dashboard_link)
futures = [client.submit(dissolve, df, date) for date, df in groups]
grouped_results = []
@@ -382,7 +1139,7 @@ def prepare_calfin(
calfin_grouped = pd.concat(grouped_results).reset_index()
# Step 2: Cumulative union (O(n) instead of O(n²))
- print("Computing cumulative unions...")
+ logger.info("Computing cumulative unions...")
cumulative_geoms = []
cumulative = None
for _, row in tqdm(calfin_grouped.iterrows(), total=len(calfin_grouped), desc="Cumulative dissolve"):
@@ -398,7 +1155,7 @@ def prepare_calfin(
agg_groups = [(date, df) for date, df in calfin_aggregated.groupby(pd.Grouper(freq=freq)) if len(df) > 0]
with Client(n_workers=n_workers, threads_per_worker=1) as client:
- print(f"Dask dashboard: {client.dashboard_link}")
+ logger.info("Dask dashboard: %s", client.dashboard_link)
futures = [
client.submit(
@@ -419,7 +1176,7 @@ def prepare_calfin(
result_filtered = [r for r in raster_results if r is not None]
# Merge and save
- print(f"Merging datasets and saving to {p_fn.resolve()}")
+ logger.info("Merging datasets and saving to %s", p_fn.resolve())
cdo = Cdo()
cdo.settbounds(
@@ -435,7 +1192,9 @@ def prepare_ismip7_forcing(
base_path: Path | str,
output_path: Path | str,
config: dict,
+ data_path: Path | str | None = None,
n_workers: int = 2,
+ staging_path: Path | str | None = None,
) -> Sequence[Path | str]:
"""
Process forcing data for all GCMs and forcings in parallel.
@@ -443,13 +1202,25 @@ def prepare_ismip7_forcing(
Parameters
----------
base_path : Path or str
- Base path to input data.
+ Base path (or URL) to the remote ISMIP7 forcing tree. Used only when
+ ``data_path`` is ``None``; otherwise downloads are skipped entirely.
output_path : Path or str
- Output directory.
+ Output directory. Only the final merged forcing files end up here.
config : dict
Configuration dictionary.
+ data_path : Path or str or None, optional
+ If given, read forcing files from this local mirror of the Globus
+ tree instead of downloading. Layout is expected to match the
+ Globus tree, with *data_path* either containing a ``GrIS/`` subdir
+ or being that ``GrIS/`` directory itself.
n_workers : int, optional
Number of dask workers, by default 2.
+ staging_path : Path or str or None, optional
+ Directory for intermediate scratch (per-variable cdo tmps and the
+ per-epoch hist/proj outputs). The whole staging tree is removed
+ after each forcing finishes, so the only artifact left on disk is
+ the final merged file in ``output_path``. Defaults to
+ ``output_path`` when omitted, matching the legacy behavior.
Returns
-------
@@ -460,29 +1231,62 @@ def prepare_ismip7_forcing(
base_path = Path(base_path)
output_path = Path(output_path)
+ if data_path is not None:
+ data_path = Path(data_path)
+ if staging_path is not None:
+ staging_path = Path(staging_path)
+ staging_path.mkdir(parents=True, exist_ok=True)
ismip7_to_pism = config["ismip7_to_pism"]
- # Build list of tasks
+ # Build list of tasks. Each ``pathway`` (``historical`` / ``ssp???``) is
+ # its own task now; the caller decides which forward run pairs them up
+ # (see run.py where ``run_hist`` uses the historical file and
+ # ``run_proj`` uses the ssp file). ``end`` is inclusive per the
+ # setup TOML convention.
tasks = []
- for gcm in config["gcms"]:
- for pathway in config["pathway"]:
- version = "v" + str(config["pathway"][pathway]["version"])
- start_year = config["pathway"][pathway]["start_year"]
- end_year = config["pathway"][pathway]["end_year"]
+ for gcm, _gcm_config in config["gcms"].items():
+ for pathway, _pathway_config in _gcm_config.items():
+ ice_sheet = config["ice_sheet"]
+ version = "v" + str(_pathway_config["version"])
+ start_year = int(_pathway_config["start"])
+ end_year = int(_pathway_config["end"])
for forcing, forcing_dict in config["forcing"].items():
short_hand = forcing_dict["short_hand"]
fields = forcing_dict["fields"]
- tasks.append((gcm, forcing, version, start_year, end_year, pathway, short_hand, fields))
+ tasks.append(
+ (
+ ice_sheet,
+ gcm,
+ forcing,
+ version,
+ pathway,
+ start_year,
+ end_year,
+ short_hand,
+ fields,
+ )
+ )
# Process in parallel using dask.distributed
with Client(n_workers=n_workers, threads_per_worker=1) as client:
- print(f"Dask dashboard: {client.dashboard_link}")
+ logger.info("Dask dashboard: %s", client.dashboard_link)
futures = []
- for gcm, forcing, version, start_year, end_year, pathway, short_hand, fields in tasks:
+ for (
+ ice_sheet,
+ gcm,
+ forcing,
+ version,
+ pathway,
+ start_year,
+ end_year,
+ short_hand,
+ fields,
+ ) in tasks:
future = client.submit(
_process_single_forcing,
+ ice_sheet,
gcm,
forcing,
base_path,
@@ -494,17 +1298,26 @@ def prepare_ismip7_forcing(
short_hand,
fields,
ismip7_to_pism,
+ data_path=data_path,
+ staging_path=staging_path,
)
futures.append(future)
# Collect results as they complete
processed_files = []
for future in as_completed(futures):
- output_file = future.result()
- print(f"Completed: {output_file}")
- processed_files.append(output_file)
+ output_files = future.result()
+ logger.info("Completed: %s", output_files)
+ processed_files.extend(output_files)
+
+ # Stamp the GrIS basin mask (from the packaged polygons) onto every generated
+ # ocean forcing file, after the dask loop.
+ ocean_files = [Path(f) for f in processed_files if "_ocean_" in Path(f).name]
+ if ocean_files:
+ logger.info("Adding basin mask to %d ocean forcing file(s)", len(ocean_files))
+ add_basins_to_ocean_files(ocean_files)
elapsed = time.perf_counter() - start_time
- print(f"Total processing time: {elapsed:.2f} seconds")
+ logger.info("Total processing time: %.2f seconds", elapsed)
return processed_files
diff --git a/pism_terra/ismip7/greenland/postprocess.py b/pism_terra/ismip7/greenland/postprocess.py
new file mode 100644
index 0000000..8212ac5
--- /dev/null
+++ b/pism_terra/ismip7/greenland/postprocess.py
@@ -0,0 +1,203 @@
+# Copyright (C) 2026 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+# pylint: disable=unused-import,unused-variable
+
+"""
+Postprocessing.
+"""
+
+import json
+import logging
+import time
+import warnings
+from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+from pathlib import Path
+
+import cf_xarray
+import geopandas as gpd
+import rioxarray
+import toml
+import xarray as xr
+from dask.distributed import Client, progress
+from pyfiglet import Figlet
+from tqdm import tqdm
+
+from pism_terra.log import setup_logging
+
+xr.set_options(keep_attrs=True)
+warnings.filterwarnings("ignore", message="invalid value encountered in cast", category=RuntimeWarning)
+warnings.filterwarnings("ignore", message="pkg_resources is deprecated", category=UserWarning)
+
+logger = logging.getLogger(__name__)
+
+
+def process_file(
+ infile: str | Path, basin_file: str | Path, client: Client, column: str = "SUBREGION1", crs: str = "EPSG:3413"
+):
+ """
+ Clip a NetCDF dataset to the glacier geometry defined in an BASIN file.
+
+ This function reads a NetCDF file containing geospatial data and clips it to the
+ geometry defined in a glacier outline file (e.g., BASIN shapefile). The clipped dataset
+ is saved to a new NetCDF file prefixed with "clipped_".
+
+ Parameters
+ ----------
+ infile : str or Path
+ Path to the NetCDF file to be clipped. Must contain x/y spatial dimensions.
+ basin_file : str or Path
+ Path to the BASIN glacier outline file (e.g., GeoPackage or shapefile) that defines
+ the geometry to clip the dataset to.
+ client : dask.Client
+ Dask client.
+ column : str, default "SUBREGION1"
+ Name of the column in ``basin_file`` used to identify basins (e.g.
+ ``"GIS"`` is selected for the merged-basin clip).
+ crs : str, default "EPSG:3413"
+ CRS code applied to the input dataset before clipping.
+ """
+
+ infile = Path(infile)
+ infile_name = infile.name
+ infile_path = infile.parent
+ clipped_file = infile_path / Path("clipped_" + infile_name)
+ scalar_file = infile_path / Path("fldsum_" + infile_name)
+
+ basin = gpd.read_file(basin_file)
+
+ start = time.time()
+ time_coder = xr.coders.CFDatetimeCoder(use_cftime=False)
+
+ ds = xr.open_dataset(
+ infile,
+ decode_timedelta=False,
+ decode_times=False,
+ chunks="auto",
+ engine="h5netcdf",
+ )
+
+ # Separate variables that lack spatial (x, y) dimensions, as rio.clip cannot handle them
+ non_spatial_vars = [var for var in ds.data_vars if "x" not in ds[var].dims or "y" not in ds[var].dims]
+ ds_non_spatial = ds[non_spatial_vars]
+ ds = ds.drop_vars(non_spatial_vars).rio.write_crs(crs).rio.set_spatial_dims(x_dim="x", y_dim="y")
+ ds = client.persist(ds)
+ progress(ds)
+
+ gis_clipped = ds.rio.clip(basin[basin[column] == "GIS"].geometry, drop=False)
+ gis_clipped = xr.merge([gis_clipped, ds_non_spatial])
+
+ logger.info("Writing %s", clipped_file)
+ comp = {"zlib": True, "complevel": 2}
+ encoding = {var: comp for var in gis_clipped.data_vars}
+ # Note: h5netcdf write garbles dim names on PISM state files (mixed
+ # 3D z_sigma vars + 0-D string pism_config). Stay on the default netcdf4
+ # engine here even though that's slower.
+ write_clipped = gis_clipped.to_netcdf(clipped_file, encoding=encoding, compute=False)
+ future_clipped = client.compute(write_clipped)
+ progress(future_clipped)
+
+ dss = []
+ for _, row in tqdm(basin.iterrows(), total=len(basin), desc="Clipping basins"):
+ ds_clipped = ds.rio.clip([row.geometry], drop=False)
+ ds_sum = ds_clipped.sum(dim=["y", "x"]).compute()
+ dss.append(ds_sum.expand_dims({"basin": [row[column]]}))
+
+ scalar = xr.concat(dss, dim="basin")
+
+ logger.info("Writing %s", scalar_file)
+ # Keep non-spatial vars (e.g. pism_config)
+ extra_vars = [v for v in ds_non_spatial.data_vars if "time" not in ds_non_spatial[v].dims]
+ if extra_vars:
+ scalar = xr.merge([scalar, ds_non_spatial[extra_vars].compute()])
+ encoding_scalar = {var: comp for var in scalar.data_vars}
+ scalar.to_netcdf(scalar_file, encoding=encoding_scalar)
+
+ end = time.time()
+ time_elapsed = end - start
+ logger.info("Time elapsed for %s: %.0fs", infile_name, time_elapsed)
+
+
+def postprocess_glacier(config_file: str | Path, n_workers: int = 4):
+ """
+ Postprocess ISMIP7 Greenland output by clipping to basin geometries.
+
+ Reads a TOML run-configuration, opens the configured ``spatial`` output
+ NetCDF, and clips it to the basin outline using a Dask client.
+
+ Parameters
+ ----------
+ config_file : str or Path
+ Path to a TOML file containing PISM run configuration with at least
+ ``[basin].outline`` and ``[output].spatial`` keys.
+ n_workers : int, optional
+ Number of Dask workers, by default 4.
+ """
+
+ config_toml = toml.load(config_file)
+ config = json.loads(json.dumps(config_toml))
+
+ start = time.time()
+ outline_file = config["basin"]["outline"]
+
+ client = Client(n_workers=n_workers, threads_per_worker=1)
+ logger.info("Dask dashboard: %s", client.dashboard_link)
+
+ for o in ["spatial"]:
+ s_file = Path(config["output"][o])
+ process_file(s_file, outline_file, client)
+
+ client.close()
+
+ end = time.time()
+ time_elapsed = end - start
+ logger.info("Time elapsed %.0fs", time_elapsed)
+
+
+def main():
+ """
+ Run main script.
+ """
+
+ # set up the option parser
+ parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+ parser.description = "Postprocess KITP Greenland."
+ parser.add_argument(
+ "--ntasks",
+ help="Sets number of tasks.",
+ type=int,
+ default=4,
+ )
+ parser.add_argument(
+ "RUN_FILE",
+ help="CONFIG TOML.",
+ nargs=1,
+ )
+
+ options, unknown = parser.parse_known_args()
+ config_file = options.RUN_FILE[0]
+ ntasks = options.ntasks
+
+ config_path = Path(config_file).resolve().parent
+ setup_logging(config_path / "postprocess.log")
+
+ postprocess_glacier(config_file, n_workers=ntasks)
+
+
+if __name__ == "__main__":
+ __spec__ = None # type: ignore
+ main()
diff --git a/pism_terra/ismip7/greenland/prepare.py b/pism_terra/ismip7/greenland/prepare.py
index e38868d..a8a6ddd 100644
--- a/pism_terra/ismip7/greenland/prepare.py
+++ b/pism_terra/ismip7/greenland/prepare.py
@@ -21,8 +21,10 @@
Prepare ISMIP7 Greenland data sets.
"""
+import logging
import os
import re
+import shutil
import time
from argparse import ArgumentParser
from pathlib import Path
@@ -42,16 +44,27 @@
from pism_terra.domain import create_domain
from pism_terra.ismip7.greenland.forcing import (
+ add_basins_to_ocean_files,
prepare_calfin,
prepare_ismip7_forcing,
prepare_observations,
)
+from pism_terra.log import setup_logging
+from pism_terra.prepare_select import add_include_argument, select_datasets
from pism_terra.raster import create_ds
from pism_terra.vector import dissolve
from pism_terra.workflow import check_xr_fully, check_xr_lazy
xr.set_options(keep_attrs=True)
+logger = logging.getLogger(__name__)
+
+# Datasets the ISMIP7 Greenland prepare can process, in execution order.
+ISMIP7_DATASETS = ["grid", "observations", "forcings", "calfin"]
+
+# Default observation NetCDF (Globus) with the boot / velocity / heat-flux inputs.
+DEFAULT_OBS_URL = "https://g-ab4495.8c185.08cc.data.globus.org/ISMIP7/Observations/Greenland/GreenlandObsISMIP7-v1.3.nc"
+
def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
"""
@@ -74,45 +87,54 @@ def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
dict[str, Any]
Results dictionary containing:
- - ``"config"``: dict
- The parsed TOML configuration used for processing.
+ - ``"config"`` : dict — parsed TOML configuration.
+ - ``"grid_file"`` : Path — generated grid NetCDF.
+ - ``"boot_file"`` : Path — observation-derived boot NetCDF.
+ - ``"heatflux_file"`` : Path — geothermal heat-flux NetCDF.
+ - ``"forcing_files"`` : sequence of Path — climate/ocean forcing files.
+ - ``"retreat_file"`` : Path — CALFIN front-retreat NetCDF.
"""
parser = ArgumentParser()
- parser.add_argument("--obs-path", default="data/obs")
parser.add_argument(
"--force-overwrite",
help="Force downloading all files.",
action="store_true",
default=False,
)
+ parser.add_argument(
+ "--data-path", help="Path to ISMIP7 data folder. If not None, use local folder instead of remote.", default=None
+ )
+ add_include_argument(parser, ISMIP7_DATASETS)
parser.add_argument("CONFIG_FILE", nargs=1)
- parser.add_argument("DATA_PATH", nargs=1)
parser.add_argument("OUTPUT_PATH", nargs=1)
args = parser.parse_args(list(argv) if argv is not None else None)
config_file = args.CONFIG_FILE[0]
- data_path = Path(args.DATA_PATH[0])
force_overwrite = args.force_overwrite
- obs_path = Path(args.obs_path)
+ data_path = Path(args.data_path) if args.data_path else None
output_path = Path(args.OUTPUT_PATH[0])
output_path.mkdir(parents=True, exist_ok=True)
+ # Intermediate scratch (cdo tmps, per-epoch hist/proj) goes here so the
+ # final ``output_path`` only carries the merged files we actually ship.
+ # Matches the ``staging`` convention used by ``pism-glacier-stage``.
+ staging_path = output_path / "staging"
+ staging_path.mkdir(parents=True, exist_ok=True)
+
+ setup_logging(output_path / "prepare.log")
+
+ selected = select_datasets(args.include, ISMIP7_DATASETS)
f = Figlet(font="standard")
banner = f.renderText("pism-terra")
- print("=" * 120)
- print(banner)
- print("=" * 120)
- print("Preparing ISMIP7 Greenland data")
- print("-" * 120)
- print("")
+ logger.info("=" * 120)
+ logger.info("\n%s", banner)
+ logger.info("=" * 120)
+ logger.info("Preparing ISMIP7 Greenland data")
+ logger.info("-" * 120)
config = toml.loads(Path(config_file).read_text("utf-8"))
- print("-" * 120)
- print("Grid File")
- print("-" * 120)
-
x_bnds = config["domain"]["x_bounds"]
y_bnds = config["domain"]["y_bounds"]
resolution_str = config["domain"]["resolution"]
@@ -121,52 +143,159 @@ def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
raise ValueError(f"Cannot parse resolution string: {resolution_str!r}")
resolution, _ = int(match.group(1)), match.group(2)
- grid_ds = create_domain(x_bnds, y_bnds, resolution)
- grid_file = output_path / Path("ismip7_greenland_grid.nc")
- encoding = {var: {"_FillValue": None} for var in list(grid_ds.data_vars) + list(grid_ds.coords)}
- grid_ds.to_netcdf(grid_file, encoding=encoding)
- check_xr_fully(grid_file)
+ # --- Grid (a dependency of the observations target grid) ---
+ grid_file = output_path / Path("pism_bedmachine_greenland_grid.nc")
+ grid_ds = None
+ if {"grid", "observations"} & set(selected):
+ logger.info("-" * 120)
+ logger.info("Grid File")
+ logger.info("-" * 120)
+ grid_ds = create_domain(x_bnds, y_bnds, resolution)
+ if "grid" in selected:
+ grid_ds.to_netcdf(grid_file)
+ check_xr_fully(grid_file)
- print("-" * 120)
- print("Calfin Glacier Fronts File")
- print("-" * 120)
+ # Observation NetCDF with the boot / velocity / heat-flux inputs, used by the
+ # observations step.
+ obs_url: str | Path = DEFAULT_OBS_URL
+ if data_path is not None:
+ obs_url = (
+ data_path / Path(config["ice_sheet"]) / Path("obs") / Path("mipkit") / Path("GreenlandObsISMIP7-v1.3.nc")
+ )
- retreat_file = prepare_calfin(
- output_path, resolution=resolution, x_bnds=x_bnds, y_bnds=y_bnds, force_overwrite=force_overwrite
- )
+ # --- Observations (boot, heatflux, velocity) ---
+ obs_files: dict[str, Any] = {}
+ if "observations" in selected:
+ logger.info("-" * 120)
+ logger.info("Boot File")
+ logger.info("-" * 120)
+ surface_dem = "s3://pism-cloud-data/dem_reconstructions/bedmachine1980_GP_reconstruction_g600.nc"
+ # When data_path is None, fall back to an obs-cache subdir under output_path
+ # so prepare_observations always has a real directory to download into.
+ obs_input_path = (
+ data_path / Path("GrIS") / Path("obs") / Path("mipkit")
+ if data_path is not None
+ else output_path / Path("obs")
+ )
+ obs_files = prepare_observations(
+ obs_url,
+ obs_input_path,
+ output_path,
+ config,
+ surface_dem=surface_dem,
+ target_grid=grid_ds,
+ force_overwrite=force_overwrite,
+ )
+ for v in obs_files.values():
+ check_xr_lazy(v)
- url = "https://g-ab4495.8c185.08cc.data.globus.org/ISMIP6/ISMIP7_Prep/Observations/Greenland/GreenlandObsISMIP7-v1.3.nc"
- print("-" * 120)
- print("Boot File")
- print("-" * 120)
- surface_dem = "s3://pism-cloud-data/dem_reconstructions/bedmachine1980_GP_reconstruction_g600.nc"
- obs_files = prepare_observations(
- url,
- obs_path,
- output_path,
- config,
- surface_dem=surface_dem,
- target_grid=grid_ds,
- force_overwrite=force_overwrite,
- )
- for v in obs_files.values():
- check_xr_lazy(v)
+ # --- Forcings ---
+ forcing_files: list = []
+ if "forcings" in selected:
+ logger.info("-" * 120)
+ logger.info("Forcings")
+ logger.info("-" * 120)
+ base_url = "https://g-ab4495.8c185.08cc.data.globus.org/ISMIP7/GrIS/"
+ forcing_files = list(
+ prepare_ismip7_forcing(
+ base_url,
+ output_path,
+ config,
+ data_path=data_path,
+ staging_path=staging_path,
+ )
+ )
+ logger.info("Forcing files: %s", forcing_files)
+
+ # --- CalFin glacier fronts ---
+ retreat_file = None
+ if "calfin" in selected:
+ logger.info("-" * 120)
+ logger.info("Calfin Glacier Fronts File")
+ logger.info("-" * 120)
+ retreat_file = prepare_calfin(
+ output_path, resolution=resolution, x_bnds=x_bnds, y_bnds=y_bnds, force_overwrite=force_overwrite
+ )
- print("-" * 120)
- print("Forcings")
- print("-" * 120)
- forcing_files = prepare_ismip7_forcing(data_path, output_path, config)
+ # Ship only the outputs produced by the selected datasets.
+ input_files: list = []
+ if "grid" in selected:
+ input_files.append(grid_file)
+ input_files += list(obs_files.values())
+ if retreat_file is not None:
+ input_files.append(retreat_file)
+ input_files += list(forcing_files)
+
+ s3_output_path = output_path / Path(config["prefix"]) / Path(config["version"])
+ s3_output_path.mkdir(parents=True, exist_ok=True)
+ logger.info("-" * 120)
+ logger.info("Copying input files to %s", s3_output_path)
+ logger.info("-" * 120)
+ for f in input_files:
+ dest = s3_output_path / Path(f).name
+ shutil.copy2(f, dest)
+ logger.info(" %s", dest)
return {
"config": config,
"grid_file": grid_file,
- "boot_file": obs_files["boot_file"],
- "heatflux_file": obs_files["heatflux_file"],
+ "boot_file": obs_files.get("boot_file"),
+ "heatflux_file": obs_files.get("heatflux_file"),
"forcing_files": forcing_files,
"retreat_file": retreat_file,
+ "obs_file": obs_files.get("obs_file"),
}
+def add_basins(argv: Sequence[str] | None = None) -> int:
+ """
+ Backfill the GrIS basin mask onto existing ISMIP7 ocean forcing files.
+
+ Console entry point (``pism-ismip7-greenland-add-basins``) that stamps the
+ ``basins`` variable (from the packaged basin polygons) onto already-generated
+ ocean forcing files without regenerating them. Each positional argument may be
+ an ocean NetCDF or a directory (scanned for ``ismip7_greenland_ocean_*.nc``);
+ only files whose name contains ``_ocean_`` are processed.
+
+ Parameters
+ ----------
+ argv : sequence of str or None, optional
+ Command-line arguments (without the program name). If ``None`` (default),
+ uses ``sys.argv``.
+
+ Returns
+ -------
+ int
+ Exit code: ``0`` on success, ``1`` if no ocean files were found.
+ """
+ logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+ parser = ArgumentParser(description="Add the GrIS basin mask to existing ISMIP7 ocean forcing files.")
+ parser.add_argument(
+ "OCEAN_FILES",
+ nargs="+",
+ help="Ocean forcing NetCDF file(s), or directories to scan for ismip7_greenland_ocean_*.nc.",
+ )
+ args = parser.parse_args(list(argv) if argv is not None else None)
+
+ candidates: list[Path] = []
+ for entry in args.OCEAN_FILES:
+ p = Path(entry)
+ if p.is_dir():
+ candidates.extend(sorted(p.glob("ismip7_greenland_ocean_*.nc")))
+ else:
+ candidates.append(p)
+ ocean_files = [p for p in candidates if "_ocean_" in p.name]
+
+ if not ocean_files:
+ logger.warning("No ocean forcing files found (need '_ocean_' in the filename).")
+ return 1
+
+ logger.info("Adding basin mask to %d ocean forcing file(s)", len(ocean_files))
+ add_basins_to_ocean_files(ocean_files)
+ return 0
+
+
def cli(argv: Sequence[str] | None = None) -> int:
"""
Console entry point.
diff --git a/pism_terra/ismip7/greenland/run.py b/pism_terra/ismip7/greenland/run.py
index d5a413f..bff02b2 100644
--- a/pism_terra/ismip7/greenland/run.py
+++ b/pism_terra/ismip7/greenland/run.py
@@ -34,42 +34,44 @@
from jinja2 import Environment, FileSystemLoader, StrictUndefined
from pyfiglet import Figlet
-from pism_terra.config import JobConfig, RunConfig, load_config, load_uq
+from pism_terra.config import JobConfig, load_config, load_uq
+from pism_terra.ismip7.experiments import resolve_counter
from pism_terra.ismip7.greenland.stage import stage
-from pism_terra.sampling import create_samples
+from pism_terra.ismip7.naming import ISMIP7Names, member_ids
+from pism_terra.sampling import generate_samples
from pism_terra.workflow import (
apply_choice_mapping,
dict2str,
- merge_model,
+ filter_overrides_by_config,
normalize_row,
sort_dict_by_key,
+ validate_pism_options,
)
# one Jinja environment for all renders
_JINJA = Environment(undefined=StrictUndefined, autoescape=False)
-def run_greenland(
+def _render_forward_run(
config_file: str | Path,
template_file: Path | str,
+ outline_file: Path | str | None,
path: str | Path = "result",
- resolution: None | str = None,
- nodes: None | int = None,
- ntasks: None | int = None,
- queue: None | str = None,
- walltime: None | str = None,
+ config_cli: dict | None = None,
debug: bool = False,
*,
uq: Mapping[str, object] | pd.Series | None = None,
sample: int | None = None,
+ pism_config_cdl: str | Path | None = None,
+ proj_overrides: Mapping[str, object] | None = None,
):
"""
- Configure and generate a PISM job script for a single glacier (ensemble-ready).
+ Configure and generate a PISM forward job script for ISMIP7 Greenland (ensemble-ready).
Reads a TOML configuration, merges optional ensemble overrides (``uq``),
renders a submission script from a Jinja2 template, and writes both the
- script and a companion TOML describing the resolved run parameters.
- Also emits a command-line string of PISM flags derived from the config and
+ script and a companion TOML describing the resolved run parameters. Also
+ emits a command-line string of PISM flags derived from the config and
overrides.
Parameters
@@ -80,81 +82,489 @@ def run_greenland(
template_file : str or pathlib.Path
Path to a Jinja2 submission template (e.g., SLURM/LSF script). The
context is populated from validated ``RunConfig`` and ``JobConfig``.
+ outline_file : str or pathlib.Path or None
+ Path to a geopandas file with the basin outline used by
+ post-processing. Pass ``None`` to record it as the literal string
+ ``"none"``.
path : str or pathlib.Path, optional
- Base output directory. A subfolder ``/`` is created with
- ``output/`` and ``run_scripts/`` subdirectories. Default is ``"result"``.
- resolution : str or None, optional
- Grid resolution (e.g., ``"200m"``). If ``None``, the value from
- ``[grid].resolution`` in the config is used.
- nodes : int or None, optional
- Node count override for the submission template. If ``None``, use config.
- ntasks : int or None, optional
- MPI task count override for the submission template/run options.
- If ``None``, use config.
- queue : str or None, optional
- Batch queue/partition override for the submission template. If ``None``,
- use config.
- walltime : str or None, optional
- Wall time override in ``HH:MM:SS``. If ``None``, use config.
+ Base output directory. ``output/`` and ``run_scripts/`` subdirectories
+ are created inside it. Default is ``"result"``.
+ config_cli : dict or None, optional
+ CLI-side overrides applied after reading the config. Recognized keys:
+ ``"resolution"`` (e.g. ``"500m"``), ``"nodes"`` (int), ``"ntasks"``
+ (int), ``"tasks"`` (int, MPI tasks per node), ``"queue"`` (str),
+ ``"walltime"`` (``HH:MM:SS``), ``"stress_balance"`` (sub-model name
+ swap, e.g. ``"sia"``), and ``"start"`` / ``"end"`` (``YYYY-MM-DD``
+ time bounds). Any value of ``None`` falls back to the config file.
+ Default is ``None`` (no overrides).
debug : bool, optional
If ``True``, skip rendering the template (leave it empty) but still
- append the constructed PISM command line to the output script.
- Default is ``False``.
+ write the resolved post-processing TOML. Default is ``False``.
uq : Mapping[str, object] or pandas.Series or None, optional
Ensemble overrides. Keys are **dotted PISM flags** (e.g.,
- ``"surface.pdd.factor_ice"``, ``"input.file"``). Values are inserted into
- the run dictionary and thus into the generated command line. If ``uq``
- contains a key ``"sample"``, it is used (when ``sample`` is not provided)
- to suffix output filenames and scripts.
+ ``"surface.pdd.factor_ice"``, ``"input.file"``). Values are inserted
+ into the run dictionary and thus into the generated command line. If
+ ``uq`` contains a key ``"sample"``, it is used (when ``sample`` is
+ not provided) to suffix output filenames and scripts.
sample : int or None, optional
Ensemble member identifier. If not provided, and ``uq`` has
``"sample"``, that value is used. The value changes the filename
- stem used for outputs (e.g., ``..._s0042``). If neither is provided,
- filenames use a descriptive ``surface/energy/stress_balance`` suffix.
+ stem used for outputs (e.g., ``..._id_0042``). If neither is
+ provided, filenames use a descriptive
+ ``surface/energy/stress_balance`` suffix.
+ pism_config_cdl : str or Path or None, optional
+ Path to a PISM CDL master config file. If provided, all run options
+ are validated against it before generating the command line.
+ proj_overrides : Mapping[str, object] or None, optional
+ Projection-only overrides applied to ``run_proj`` after it is
+ copied from ``run_hist``. Same schema as ``uq`` (dotted PISM
+ flags). Used to point ``atmosphere.given.file`` etc. at the
+ projection-epoch forcing file while ``run_hist`` keeps the
+ historical file. Default is ``None`` (no proj-only overrides).
Raises
------
ValueError
If configuration validation fails upstream (e.g., via Pydantic models),
or if provided overrides are of incompatible types.
+ """
+
+ outline_file = str(Path(outline_file).resolve()) if (outline_file is not None) else "none"
+ cfg = load_config(config_file)
+
+ config_cli = config_cli or {}
+ resolution = config_cli.get("resolution")
+ if resolution:
+ resolution = re.sub(r"\s+", "", resolution)
+
+ # update GridConfig and force dx/dy to be derived from the new resolution
+ cfg.grid.resolution = resolution
+ cfg.grid.dx = None
+ cfg.grid.dy = None
+
+ path = Path(path)
+ path.mkdir(parents=True, exist_ok=True)
+ log_path = path / Path("logs")
+ log_path.mkdir(parents=True, exist_ok=True)
+ output_path = path / Path("output")
+ output_path.mkdir(parents=True, exist_ok=True)
+ scalar_path = output_path / Path("scalar")
+ scalar_path.mkdir(parents=True, exist_ok=True)
+ spatial_path = output_path / Path("spatial")
+ spatial_path.mkdir(parents=True, exist_ok=True)
+ state_path = output_path / Path("state")
+ state_path.mkdir(parents=True, exist_ok=True)
+
+ run_hist = {}
+ for section in (
+ "geometry",
+ "calving",
+ "iceflow",
+ "reporting",
+ "input",
+ "time_stepping",
+ ):
+ run_hist.update(getattr(cfg, section))
+ run_hist.update(cfg.atmosphere.selected())
+ run_hist.update(cfg.bed_deformation.selected())
+ run_hist.update(cfg.energy.selected())
+ run_hist.update(cfg.ocean.selected())
+ run_hist.update(cfg.frontal_melt.selected())
+ run_hist.update(cfg.grid.as_params())
+ run_hist.update(cfg.hydrology.selected())
+ run_hist.update(cfg.run_info.as_params())
+ run_hist.update(cfg.surface.selected())
+ run_hist.update(cfg.stress_balance.selected())
+ run_hist.update(cfg.time.as_params())
+ # PETSc / blatter solver knobs from [solver.forward]. Matches what the
+ # inverse runner does for the prior pism call; see also pism_terra.glacier.run.
+ run_hist.update(cfg.solver.get("forward", {}))
- Notes
- -----
- - The Jinja2 context is populated from validated ``RunConfig`` and
- ``JobConfig`` (config values) plus any CLI overrides provided here
- for ``ntasks``, ``nodes``, ``queue``, ``walltime``.
- - ``uq`` overrides are merged **after** reading the config; they can set or
- replace any dotted PISM flag (e.g., swapping input or forcing files).
- - The function attempts to open NetCDF inputs referenced by keys ending
- with ``.file`` (excluding ``output.*``) using ``xarray.open_dataset`` and
- prints a ✓/✗ check; it does not stop the run on failure.
-
- Examples
- --------
- Basic use with config and template:
-
- >>> run_glacier(
- ... rgi_id="RGI2000-v7.0-C-01-04374",
- ... config_file="config/init_stampede3.toml",
- ... template_file="templates/stampede3.j2",
- ... path="result",
- ... )
-
- Ensemble member with overrides from a pandas row (e.g., Latin Hypercube):
-
- >>> row = df_samples.loc[17] # contains dotted keys + 'sample'
- >>> run_glacier(
- ... rgi_id="RGI2000-v7.0-C-01-04374",
- ... config_file="config/init_stampede3.toml",
- ... template_file="templates/stampede3.j2",
- ... uq=row, # dotted PISM flags to override
- ... sample=None, # will be inferred from row['sample'] if present
- ... ntasks=112, # optional template/run override
- ... )
+ template_file = Path(template_file)
+ env = Environment(loader=FileSystemLoader(template_file.parent))
+ template = env.get_template(template_file.name)
+
+ # CLI overrides for time bounds. ``cfg.time`` is a TimeConfig pydantic
+ # model with field names ``time_start`` / ``time_end`` (aliased to the
+ # dotted ``"time.start"`` / ``"time.end"``), so attribute assignment is
+ # required. Drop the prior dotted entry from ``run_hist`` and re-apply via
+ # ``as_params()`` so the new value lands cleanly.
+ _start = config_cli.get("start")
+ _end = config_cli.get("end")
+ if _start is not None:
+ run_hist.pop("time.start", None)
+ cfg.time.time_start = _start
+ run_hist.update(cfg.time.as_params())
+ if _end is not None:
+ run_hist.pop("time.end", None)
+ cfg.time.time_end = _end
+ run_hist.update(cfg.time.as_params())
+
+ start = cfg.model_dump(by_alias=True)["time"]["time.start"]
+ end = cfg.model_dump(by_alias=True)["time"]["time.end"]
+
+ if resolution is None:
+ resolution = cfg.model_dump(by_alias=True)["grid"]["resolution"]
+ # CLI override for the stress-balance model. Drop the previous model's
+ # options from ``run_hist`` first so leftover keys (e.g. blatter.*) don't
+ # leak into e.g. a sia run.
+ stress_balance = config_cli.get("stress_balance")
+ if stress_balance is not None:
+ for old_key in cfg.stress_balance.selected():
+ run_hist.pop(old_key, None)
+ cfg.stress_balance.model = stress_balance
+ run_hist.update(cfg.stress_balance.selected())
+ stress_balance = cfg.model_dump(by_alias=True)["stress_balance"]["model"]
+
+ energy = cfg.model_dump(by_alias=True)["energy"]["model"]
+ surface = cfg.model_dump(by_alias=True)["surface"]["model"]
+
+ # ``InfoConfig.as_params()`` deliberately drops the ISMIP7 naming-only
+ # fields (see ``_PISM_FIELDS`` in config.py) — grab ``experiment`` off
+ # the pydantic model directly.
+ experiment = cfg.run_info.experiment or "none"
+ if sample is None:
+ name_options = f"surface_{surface}_energy_{energy}_stress_balance_{stress_balance}"
+ else:
+ name_options = f"id_{sample}_{experiment}"
+
+ uq_clean = normalize_row(uq) if uq is not None else {}
+ # Prefer explicit `sample` arg; else default from uq['sample']
+ if sample is None and "sample" in uq_clean:
+ try:
+ sample = int(uq_clean["sample"])
+ except Exception:
+ pass
+
+ # Remove 'sample' from flag overrides; drop any key not in the config-derived
+ # run_hist dict (e.g., surface.debm_simple.std_dev.file when surface.model == "pdd").
+ overrides = {k: v for k, v in uq_clean.items() if k != "sample"}
+ overrides, skipped = filter_overrides_by_config(overrides, run_hist.keys())
+ if skipped:
+ print(f"Skipping uq overrides not in config: {skipped}")
+ # Apply to runtime dict (these should be dotted PISM flags)
+ run_hist.update(overrides)
+
+ run_hist.pop("time.end", None)
+ run_hist.update({"time.end": "2015-01-01"})
+ # Match InfoConfig._quote()'s output shape so both hist and proj write
+ # ``run_info.experiment`` the same way (see run_proj below).
+ run_hist.update({"run_info.experiment": '"historical"'})
+
+ # ISMIP7 submission naming (conventions doc section 8): when output.ISMIP is
+ # set, write the spatial/scalar outputs into the
+ # ///// tree with conforming names.
+ # PISM expands the {var} placeholder, so the spatial output is already one
+ # conforming file per variable. The scalar time series stays a single file
+ # (its per-variable split is deferred to post-processing). The state/restart
+ # file is not an ISMIP7 product, so it stays in state/.
+ #
+ # The forward call splits into an ``hist`` PISM invocation and a follow-on
+ # ``proj`` invocation; each needs its own conforming filenames because the
+ # ``experiment_id`` and ``time_range`` differ. Precompute the parts that
+ # only depend on the ensemble member (gcm / ism member / set counter) and
+ # then generate the two file triples via ``_output_files``.
+ # ISMIP7 Core Experiment counter (e.g. "C003"), if this run is counter-driven.
+ # It fixes the ISMIP7 ``set_counter`` and selects which of the two forward legs
+ # is the submission product (the other leg gets flat filenames). ``None`` keeps
+ # the legacy behavior: both legs use ISMIP7 names when ``output.ISMIP`` is set.
+ counter = cfg.run_info.counter
+ product_leg: str | None = None
+ if counter:
+ product_leg = resolve_counter(counter).product_leg
+
+ use_ismip = str(run_hist.get("output.ISMIP", "no")).strip().strip("\"'").lower() in ("yes", "true", "1")
+ ismip7_ctx: dict | None = None
+ if use_ismip:
+ ri = cfg.run_info
+ missing = [a for a in ("domain", "group", "ism", "set_id", "experiment") if not getattr(ri, a)]
+ if missing:
+ raise SystemExit(f"output.ISMIP requires run_info fields: {', '.join(f'run_info.{m}' for m in missing)}")
+ gcms = cfg.campaign.as_params().get("gcms") or []
+ esm_id = str(sample) if sample is not None else (gcms[0] if gcms else "none")
+ member_index = gcms.index(esm_id) if esm_id in gcms else 0
+ set_counter, ism_member, forcing_member = member_ids(str(ri.set_id), member_index)
+ # A counter-driven run uses its protocol counter as the ISMIP7 set_counter
+ # (member_ids still supplies the CORE m001/f001 member ids).
+ if counter:
+ set_counter = counter
+ ismip7_ctx = {
+ "domain_id": str(ri.domain),
+ "source_id": str(ri.group),
+ "ism_id": str(ri.ism),
+ "ism_member_id": ism_member,
+ "esm_id": esm_id,
+ "forcing_member_id": forcing_member,
+ "set_id": str(ri.set_id),
+ "set_counter": set_counter,
+ }
+
+ def _output_files(experiment_id: str, start_str: str, end_str: str, *, ismip7: bool) -> tuple[Path, Path, Path]:
+ """
+ Build the (state, spatial, scalar) file triple for one PISM invocation.
+
+ Uses ISMIP7-conforming names under ``//…`` when
+ ``output.ISMIP`` is enabled; falls back to the flat
+ ``g___`` layout under the usual ``scalar/`` /
+ ``spatial/`` / ``state/`` subdirectories otherwise. The state file
+ always stays in ``state/`` (not an ISMIP7 product).
+
+ Parameters
+ ----------
+ experiment_id : str
+ ISMIP7 experiment identifier (e.g. ``"historical"`` or
+ ``"ssp370"``). Only used when ``output.ISMIP`` is enabled; feeds
+ into both the directory tree and the encoded filename stem.
+ start_str : str
+ Start of the simulated interval as ``YYYY-MM-DD``. Contributes
+ the leading year of the ``time_range`` in the ISMIP7 stem, and
+ appears verbatim in the flat-layout fallback filename.
+ end_str : str
+ End of the simulated interval as ``YYYY-MM-DD``. Contributes
+ the trailing year of ``time_range`` (with a ``-1`` correction
+ when the timestamp lands exactly on Jan 1 so the range reads
+ inclusive on the source-year side).
+ ismip7 : bool
+ Whether *this* leg is the ISMIP7 submission product. When ``False``
+ (or ``output.ISMIP`` is off) the flat ``spatial_``/``scalar_`` layout
+ is used even if ``ismip7_ctx`` is populated, so the non-product leg of
+ a counter-driven run does not land in the submission tree.
+
+ Returns
+ -------
+ tuple of pathlib.Path
+ ``(state, spatial, scalar)`` — absolute paths for PISM's
+ ``output.file``, ``output.spatial.file`` (with an unexpanded
+ ``{var}`` placeholder that PISM fills in per variable), and
+ ``output.scalar.file``.
+ """
+ tag = f"g{resolution}_{name_options}_{start_str}_{end_str}"
+ state = state_path / Path(f"state_{tag}.nc")
+ if ismip7_ctx is None or not ismip7:
+ spatial = spatial_path / Path(f"spatial_g{resolution}_{name_options}_{{var}}_{start_str}_{end_str}.nc")
+ scalar = scalar_path / Path(f"scalar_g{resolution}_{name_options}_{start_str}_{end_str}.nc")
+ return state, spatial, scalar
+ end_ts = pd.Timestamp(end_str)
+ last_year = end_ts.year - 1 if (end_ts.month == 1 and end_ts.day == 1) else end_ts.year
+ time_range = f"{pd.Timestamp(start_str).year}-{last_year}"
+ names = ISMIP7Names(experiment_id=experiment_id, time_range=time_range, **ismip7_ctx)
+ ismip7_dir = names.directory(output_path)
+ ismip7_dir.mkdir(parents=True, exist_ok=True)
+ spatial = ismip7_dir / names.filename("{var}")
+ scalar = ismip7_dir / f"scalar_{names.stem()}.nc"
+ return state, spatial, scalar
+
+ # Which leg is the ISMIP7 submission product: for a counter-driven run only the
+ # designated leg gets ISMIP7 names (the other is an internal continuation with
+ # flat names); legacy runs (product_leg is None) keep both legs on ISMIP7 names.
+ hist_ismip7 = product_leg in (None, "historical")
+ proj_ismip7 = product_leg in (None, "projection")
+ # Historical experiments (C001/C002) are historical-only: skip the projection
+ # continuation (and its forcing) entirely (product_leg == "historical").
+ run_projection = product_leg != "historical"
+ state_hist, spatial_hist, scalar_hist = _output_files("historical", start, "2015-01-01", ismip7=hist_ismip7)
+ proj_experiment = str(cfg.run_info.experiment) if cfg.run_info.experiment else "none"
+
+ run_hist.update(
+ {
+ "output.file": state_hist.resolve(),
+ "output.spatial.file": spatial_hist.resolve(),
+ "output.scalar.file": scalar_hist.resolve(),
+ }
+ )
+
+ if pism_config_cdl is not None:
+ validate_pism_options(run_hist, pism_config_cdl)
+
+ run_hist_str = dict2str(sort_dict_by_key(run_hist))
+
+ # Projection continuation leg. Skipped entirely for historical-only
+ # experiments (C001/C002), whose projection forcing is neither staged nor run;
+ # ``run_proj_str`` stays empty so the template omits the projection invocation.
+ run_proj_str = ""
+ if run_projection:
+ # Projection end comes from the config (time.end), which the counter
+ # resolver sets from the Core Experiment's proj_end_year (2100 or 2300).
+ state_proj, spatial_proj, scalar_proj = _output_files(proj_experiment, "2015-01-01", end, ismip7=proj_ismip7)
+
+ run_proj = run_hist.copy()
+ # Restore run_info.experiment to the projection value (run_hist has it
+ # forced to "historical"); the rest of run_info survives the copy.
+ # ``InfoConfig.as_params()`` deliberately drops the ISMIP7 naming-only
+ # fields (domain / set / ism / experiment) so they don't leak into the
+ # PISM command; we want ``run_info.experiment`` to survive, so bypass
+ # the filter with a direct assignment, quoted the same way as_params()
+ # would if it emitted the field.
+ run_proj.update(cfg.run_info.as_params())
+ if cfg.run_info.experiment:
+ run_proj["run_info.experiment"] = f'"{cfg.run_info.experiment}"'
+ run_proj.update({"input.file": state_hist.resolve()})
+ run_proj.pop("time.start", None)
+ run_proj.update({"time.start": "2015-01-01"})
+ run_proj.pop("time.end", None)
+ run_proj.update({"time.end": end})
+ run_proj.pop("input.bootstrap", None)
+ run_proj.pop("input.regrid.file", None)
+ run_proj.pop("input.regrid.vars", None)
+ run_proj.update(
+ {
+ "output.file": state_proj.resolve(),
+ "output.spatial.file": spatial_proj.resolve(),
+ "output.scalar.file": scalar_proj.resolve(),
+ }
+ )
+ # Projection-epoch file paths supplied by ``_run()`` (climate / ocean
+ # / gradient) — filtered against ``run_proj`` so a mis-typed key from
+ # the caller doesn't silently vanish.
+ if proj_overrides:
+ proj_clean = {k: v for k, v in dict(proj_overrides).items() if k != "sample"}
+ proj_clean, proj_skipped = filter_overrides_by_config(proj_clean, run_proj.keys())
+ if proj_skipped:
+ print(f"Skipping proj overrides not in config: {proj_skipped}")
+ run_proj.update(proj_clean)
+ run_proj_str = dict2str(sort_dict_by_key(run_proj))
+
+ job_opts = JobConfig(**cfg.job.model_dump())
+
+ params = {
+ **job_opts.model_dump(exclude_none=True, by_alias=True),
+ }
+
+ job_kwargs = {
+ k: v
+ for k, v in {
+ "nodes": config_cli.get("nodes"),
+ "ntasks": config_cli.get("ntasks"),
+ "queue": config_cli.get("queue"),
+ "output_path": log_path.resolve(),
+ "tasks": config_cli.get("tasks"),
+ "walltime": config_cli.get("walltime"),
+ }.items()
+ if v is not None
+ }
+ if job_kwargs:
+ params.update(JobConfig(**job_kwargs).as_params())
+
+ params.update({"run_hist_str": run_hist_str})
+ params.update({"run_proj_str": run_proj_str})
+
+ # Point the compliance checker at this run's actual ISMIP7 submission
+ # directory (output//////) rather than a
+ # hardcoded path. The directory depends only on the ismip7_ctx identity fields
+ # (experiment_id/time_range don't affect it), and both forward legs write into
+ # it. When ISMIP7 naming is off there is no submission tree, so leave it empty.
+ if ismip7_ctx is not None:
+ submission_dir = ISMIP7Names(experiment_id=proj_experiment, time_range="", **ismip7_ctx).directory(
+ output_path.resolve()
+ )
+ ism_checker_str = f"ismip7-compliance-checker --source-path {submission_dir}/ --variable-list ismip7"
+ # Split each ISMIP7 product scalar file into per-variable diagnostics via
+ # the packaged post-processing script. The product leg's scalar is used:
+ # scalar_hist for historical experiments (C001/C002), scalar_proj for the
+ # projection experiments; a legacy non-counter run treats both legs as
+ # products (hist_ismip7 and proj_ismip7 both True) and post-processes both.
+ post_script = Path(__file__).resolve().parents[2] / "data" / "postprocess_ismip7_scalar.sh"
+ post_scalars = []
+ if hist_ismip7:
+ post_scalars.append(scalar_hist)
+ if run_projection and proj_ismip7:
+ post_scalars.append(scalar_proj)
+ post_scalar_str = "\n".join(f"bash {post_script} {s.resolve()}" for s in post_scalars)
+ else:
+ ism_checker_str = ""
+ post_scalar_str = ""
+
+ params.update({"ism_checker_str": ism_checker_str})
+ params.update({"post_scalar_str": post_scalar_str})
+
+ rendered_script = "" if debug else template.render(params)
+
+ run_script_path = path / Path("run_scripts")
+ run_script_path.mkdir(parents=True, exist_ok=True)
+
+ run_script = run_script_path / Path(f"submit_g{resolution}_{name_options}.sh")
+
+ # Save or print the output
+ run_script.write_text(rendered_script)
+
+ print(f"\nJob script written to {run_script.resolve()}\n")
+
+
+def _render_inverse_run(
+ config_file: str | Path,
+ template_file: Path | str,
+ outline_file: Path | str | None,
+ path: str | Path = "result",
+ config_cli: dict | None = None,
+ debug: bool = False,
+ *,
+ uq: Mapping[str, object] | pd.Series | None = None,
+ sample: int | None = None,
+ pism_config_cdl: str | Path | None = None,
+ proj_overrides: ( # pylint: disable=unused-argument
+ Mapping[str, object] | None
+ ) = None, # accepted for signature parity with _render_forward_run; ignored
+):
+ """
+ Configure and generate a PISM inverse job script for ISMIP7 Greenland (ensemble-ready).
+
+ Same interface as :func:`_render_forward_run` but also builds an ``inv``
+ dict of ``inverse.*`` / ``stress_balance.*`` flags that the Jinja2
+ template can render as a ``pismi`` command line via ``inv_str``. Output
+ files mirror the forward layout with an additional ``inverse/``
+ subdirectory under ``output/``.
+
+ Parameters
+ ----------
+ config_file : str or pathlib.Path
+ Path to the PISM configuration TOML (contains ``run``, ``grid``,
+ ``time``, ``surface``, ``energy``, ``stress_balance``, ``inverse``).
+ template_file : str or pathlib.Path
+ Path to a Jinja2 submission template. The context includes both
+ ``run_str`` (forward command line) and ``inv_str`` (inverse command
+ line) so a single template can launch the prior + pismi pair.
+ outline_file : str or pathlib.Path or None
+ Path to a geopandas file with the basin outline used by
+ post-processing. Pass ``None`` to record it as the literal string
+ ``"none"``.
+ path : str or pathlib.Path, optional
+ Base output directory. ``output/`` (with an extra ``inverse/``
+ subdirectory) and ``run_scripts/`` are created inside it. Default
+ is ``"result"``.
+ config_cli : dict or None, optional
+ CLI-side overrides applied after reading the config. See
+ :func:`_render_forward_run` for the recognized keys. Default is
+ ``None`` (no overrides).
+ debug : bool, optional
+ If ``True``, skip rendering the template (leave it empty) but still
+ write the resolved post-processing TOML. Default is ``False``.
+ uq : Mapping[str, object] or pandas.Series or None, optional
+ Ensemble overrides. Keys are dotted PISM flags belonging to either
+ the forward (``run``) or inverse (``inv``) dict; each key is routed
+ to the dict that owns it, so e.g. ``inverse.file`` propagates into
+ ``inv_str``.
+ sample : int or None, optional
+ Ensemble member identifier. If not provided, and ``uq`` has
+ ``"sample"``, that value is used. Changes the filename stem used
+ for outputs (e.g., ``..._id_0042``).
+ pism_config_cdl : str or pathlib.Path or None, optional
+ Path to a PISM CDL master config file. If provided, all forward
+ run options are validated against it before generating the
+ command line.
+ proj_overrides : Mapping[str, object] or None, optional
+ Accepted for signature parity with :func:`_render_forward_run`
+ and **ignored** here — the inverse workflow doesn't split the
+ forward run into hist/proj epochs, so there's no ``run_proj`` to
+ apply projection-only overrides to. ``_run()`` always passes the
+ keyword when it calls a renderer, so the parameter has to exist
+ even though it's a no-op. Default is ``None``.
"""
+ outline_file = str(Path(outline_file).resolve()) if (outline_file is not None) else "none"
cfg = load_config(config_file)
+ config_cli = config_cli or {}
+ resolution = config_cli.get("resolution")
if resolution:
resolution = re.sub(r"\s+", "", resolution)
@@ -175,11 +585,12 @@ def run_greenland(
spatial_path.mkdir(parents=True, exist_ok=True)
state_path = output_path / Path("state")
state_path.mkdir(parents=True, exist_ok=True)
+ inv_path = output_path / Path("inverse")
+ inv_path.mkdir(parents=True, exist_ok=True)
run = {}
for section in (
"geometry",
- "ocean",
"calving",
"iceflow",
"reporting",
@@ -189,6 +600,7 @@ def run_greenland(
run.update(getattr(cfg, section))
run.update(cfg.atmosphere.selected())
run.update(cfg.energy.selected())
+ run.update(cfg.ocean.selected())
run.update(cfg.frontal_melt.selected())
run.update(cfg.grid.as_params())
run.update(cfg.hydrology.selected())
@@ -196,21 +608,79 @@ def run_greenland(
run.update(cfg.surface.selected())
run.update(cfg.stress_balance.selected())
run.update(cfg.time.as_params())
+ # Forward solver knobs ([solver.forward]) drive the forward pism call.
+ run.update(cfg.solver.get("forward", {}))
+
+ inv: dict = {}
+ inv.update(getattr(cfg, "iceflow"))
+ inv.update(getattr(cfg, "inverse"))
+ # Inverse solver knobs ([solver.inverse]) drive the pismi call.
+ inv.update(cfg.solver.get("inverse", {}))
+
+ # cfg.stress_balance.selected() carries everything the forward run needs
+ # (model options + PETSc solver knobs like bp_* / inv_adj_*). The pismi
+ # call only needs the ``stress_balance.*`` dotted options; the solver
+ # flags are picked up by the prior pism call (and inherited from the
+ # state file). Filter so inv_str stays minimal.
+ inv.update({k: v for k, v in cfg.stress_balance.selected().items() if k.startswith("stress_balance.")})
+
+ # pismi runs the (Blatter) stress balance during the inversion, so it needs the
+ # same energy model and flow law as the forward prior. These live in
+ # cfg.energy.selected() (e.g. energy.model and stress_balance.*.flow_law =
+ # gpbld); without them pismi would silently fall back to PISM's default flow
+ # law, making the inversion inconsistent with the forward runs.
+ inv.update(cfg.energy.selected())
template_file = Path(template_file)
env = Environment(loader=FileSystemLoader(template_file.parent))
template = env.get_template(template_file.name)
+ # CLI overrides for time bounds. ``cfg.time`` is a TimeConfig pydantic
+ # model with field names ``time_start`` / ``time_end`` (aliased to the
+ # dotted ``"time.start"`` / ``"time.end"``), so we set attributes, not
+ # items. We drop the prior value from ``run`` first and re-apply via
+ # ``as_params()`` so the dotted alias replaces cleanly.
+ _start = config_cli.get("start")
+ _end = config_cli.get("end")
+ if _start is not None:
+ run.pop("time.start", None)
+ cfg.time.time_start = _start
+ run.update(cfg.time.as_params())
+ if _end is not None:
+ run.pop("time.end", None)
+ cfg.time.time_end = _end
+ run.update(cfg.time.as_params())
+
start = cfg.model_dump(by_alias=True)["time"]["time.start"]
end = cfg.model_dump(by_alias=True)["time"]["time.end"]
- writer = cfg.model_dump()["run"]["writer"] if (cfg.model_dump()["run"]["writer"] is not None) else ""
if resolution is None:
resolution = cfg.model_dump(by_alias=True)["grid"]["resolution"]
+ # CLI override for the stress-balance model. Apply it to BOTH the forward
+ # prior (``run``) and the pismi inversion (``inv``). ``inv`` was populated
+ # with the config's default stress-balance options above, so without syncing
+ # it here the inversion would keep using that model (e.g. blatter) even when
+ # the CLI selects hybrid. ``inv`` only carried the ``stress_balance.*`` subset
+ # (not the bp_* solver flags), and the ``[iceflow]`` stress_balance.* keys are
+ # not part of ``selected()``, so popping the old model's keys leaves them intact.
+ stress_balance = config_cli.get("stress_balance")
+ if stress_balance is not None:
+ old_selected = cfg.stress_balance.selected()
+ for old_key in old_selected:
+ run.pop(old_key, None)
+ if old_key.startswith("stress_balance."):
+ inv.pop(old_key, None)
+ cfg.stress_balance.model = stress_balance
+ new_selected = cfg.stress_balance.selected()
+ run.update(new_selected)
+ inv.update({k: v for k, v in new_selected.items() if k.startswith("stress_balance.")})
stress_balance = cfg.model_dump(by_alias=True)["stress_balance"]["model"]
+
energy = cfg.model_dump(by_alias=True)["energy"]["model"]
surface = cfg.model_dump(by_alias=True)["surface"]["model"]
+ # Note: the inverse run uses flat output names and is not an ISMIP7 product,
+ # so run_info.experiment is not needed here (unlike the forward render).
if sample is None:
name_options = f"surface_{surface}_energy_{energy}_stress_balance_{stress_balance}"
else:
@@ -224,10 +694,18 @@ def run_greenland(
except Exception:
pass
- # Remove 'sample' from flag overrides
- overrides = {k: v for k, v in uq_clean.items() if k != "sample"}
- # Apply to runtime dict (these should be dotted PISM flags)
- run.update(overrides)
+ # Drop any uq key that isn't in either the ``run`` or ``inv`` dicts (e.g.
+ # surface.debm_simple.std_dev.file when surface.model == "pdd"). ``inverse.*``
+ # keys live in ``inv`` only, so filtering against ``run.keys()`` alone would
+ # silently drop them.
+ all_overrides = {k: v for k, v in uq_clean.items() if k != "sample"}
+ run_overrides, _ = filter_overrides_by_config(all_overrides, run.keys())
+ inv_overrides, _ = filter_overrides_by_config(all_overrides, inv.keys())
+ skipped = [k for k in all_overrides if k not in run and k not in inv]
+ if skipped:
+ print(f"Skipping uq overrides not in config: {skipped}")
+ run.update(run_overrides)
+ inv.update(inv_overrides)
scalar_file = scalar_path / Path(f"scalar_g{resolution}_{name_options}_{start}_{end}.nc")
spatial_file = spatial_path / Path(f"spatial_g{resolution}_{name_options}_{start}_{end}.nc")
@@ -235,38 +713,45 @@ def run_greenland(
run.update(
{
"output.file": state_file.resolve(),
- "output.spatial.file": spatial_file.resolve(),
"output.scalar.file": scalar_file.resolve(),
+ "output.spatial.file": spatial_file.resolve(),
}
)
- run_str = dict2str(sort_dict_by_key(run)) + f" {writer}"
+ if pism_config_cdl is not None:
+ validate_pism_options(run, pism_config_cdl)
+
+ run_str = dict2str(sort_dict_by_key(run))
+
+ inv_file = inv_path / Path(f"inv_g{resolution}_{name_options}_{start}_{end}.nc")
+ # Feed the forward run's state file into pismi as its input.
+ inv.update({"input.file": state_file.resolve()})
+ inv.update({"o": inv_file.resolve()})
+ inv_str = dict2str(sort_dict_by_key(inv))
- run_opts = RunConfig(**cfg.run.model_dump())
job_opts = JobConfig(**cfg.job.model_dump())
params = {
- **run_opts.model_dump(exclude_none=True, by_alias=True),
**job_opts.model_dump(exclude_none=True, by_alias=True),
}
- # run_opts comes from your config; ntasks comes from CLI (or None)
- active_run_opts = merge_model(run_opts, ntasks=ntasks)
-
- # Use this ONE source to update params and to compute mpi_str
- run_params = active_run_opts.as_params()
- params.update(run_params)
- mpi_str = run_params["mpi"] # guaranteed consistent with ntasks override
-
job_kwargs = {
k: v
- for k, v in {"queue": queue, "walltime": walltime, "nodes": nodes, "output_path": log_path.resolve()}.items()
+ for k, v in {
+ "nodes": config_cli.get("nodes"),
+ "ntasks": config_cli.get("ntasks"),
+ "queue": config_cli.get("queue"),
+ "output_path": log_path.resolve(),
+ "tasks": config_cli.get("tasks"),
+ "walltime": config_cli.get("walltime"),
+ }.items()
if v is not None
}
if job_kwargs:
params.update(JobConfig(**job_kwargs).as_params())
run_toml = {
+ "basin": {"basin": "Mouginot/Rignot", "outline": outline_file},
"output": {
"spatial": str(spatial_file.resolve()),
"state": str(state_file.resolve()),
@@ -280,276 +765,181 @@ def run_greenland(
with open(post_file, "w", encoding="utf-8") as toml_file:
toml.dump(run_toml, toml_file)
- prefix = f"{mpi_str} {cfg.run.executable} "
- postfix = "# End of script"
+ params.update({"run_str": run_str})
+ params.update({"inv_str": inv_str})
+ params.update({"post_script": "pism-ismip7-greenland-postprocess"})
+ params.update({"post_file": post_file})
rendered_script = "" if debug else template.render(params)
- rendered_script += f"\n\n{prefix}{run_str}\n\n{postfix}"
run_script_path = path / Path("run_scripts")
run_script_path.mkdir(parents=True, exist_ok=True)
run_script = run_script_path / Path(f"submit_g{resolution}_{name_options}_{start}_{end}.sh")
- # Save or print the output
run_script.write_text(rendered_script)
print(f"\nJob script written to {run_script.resolve()}\n")
+ print(f"Postprocessing script written to {post_file.resolve()}\n")
-def run_single():
- """
- Run single glacier.
+def _nullable_string(argument_string: str) -> str | None:
"""
+ Treat the literal CLI argument ``"none"`` as Python ``None``.
- # set up the option parser
- parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
- parser.description = "Run ISMIP7 Greenland."
- parser.add_argument(
- "--force-overwrite",
- help="Force downloading all files.",
- action="store_true",
- default=False,
- )
- parser.add_argument(
- "--output-path",
- help="Base path to save all files to. Files will be saved in `f'{out_path}/{RGI_ID}/output/'`.",
- type=str,
- default="data",
- )
- parser.add_argument(
- "--queue",
- help="Overrides queue in config file.",
- type=str,
- default=None,
- )
- parser.add_argument(
- "--ntasks",
- help="Overrides ntatsks in config file.",
- type=int,
- default=None,
- )
- parser.add_argument(
- "--nodes",
- help="Overrides nodes in config file.",
- type=int,
- default=None,
- )
- parser.add_argument(
- "--walltime",
- help="Overrides walltime in config file.",
- type=str,
- default=None,
- )
- parser.add_argument(
- "--resolution",
- help="Override horizontal grid resolution.",
- type=str,
- default=None,
- )
- parser.add_argument(
- "--debug",
- help="Debug or testing mode, do not write template, just the run command.",
- action="store_true",
- default=False,
- )
- parser.add_argument(
- "CONFIG_FILE",
- help="CONFIG TOML.",
- nargs=1,
- )
- parser.add_argument(
- "TEMPLATE_FILE",
- help="TEMPLATE J2.",
- nargs=1,
- )
-
- options, _ = parser.parse_known_args()
- force_overwrite = options.force_overwrite
- path = options.output_path
- config_file = options.CONFIG_FILE[0]
- template_file = options.TEMPLATE_FILE[0]
- resolution = options.resolution
- debug = options.debug
- queue = options.queue
- ntasks = options.ntasks
- nodes = options.nodes
- walltime = options.walltime
-
- path = Path(path)
- path.mkdir(parents=True, exist_ok=True)
- input_path = path / Path("input")
- input_path.mkdir(parents=True, exist_ok=True)
- output_path = path / Path("output")
- output_path.mkdir(parents=True, exist_ok=True)
+ Lets job submission systems that can't omit arguments pass a sentinel
+ string instead of dropping the flag. Mirrors
+ :func:`pism_terra.glacier.run._nullable_string`.
- cfg = load_config(config_file)
- campaign_config = cfg.campaign.as_params()
+ Parameters
+ ----------
+ argument_string : str
+ Argument string to parse.
+
+ Returns
+ -------
+ str or None
+ ``None`` if the argument is the case-insensitive literal ``"none"``,
+ otherwise the argument unchanged.
+ """
+ if argument_string.strip().lower() == "none":
+ return None
+ return argument_string
- bucket = campaign_config["bucket"]
- prefix = campaign_config["prefix"]
-
- df = stage(campaign_config, bucket=bucket, prefix=prefix, path=path, force_overwrite=force_overwrite)
-
- default = {
- "input.file": df["boot_file"].iloc[0],
- "input.regrid.file": df["regrid_file"].iloc[0],
- "frontal_melt.routing.file": df["frontal_melt_file"].iloc[0],
- "geometry.front_retreat.prescribed.file": df["retreat_file"].iloc[0],
- "grid.file": df["grid_file"].iloc[0],
- "energy.bedrock_thermal.file": df["heatflux_file"].iloc[0],
- "atmosphere.given.file": df["climate_file"].iloc[0],
- "surface.given.file": df["climate_file"].iloc[0],
- "hydrology.surface_input.file": df["surface_input_file"].iloc[0],
- "ocean.th.file": df["ocean_file"].iloc[0],
- }
- f = Figlet(font="standard")
- banner = f.renderText("pism-terra")
- print("=" * 80)
- print(banner)
- print("=" * 80)
- print("Generate Run for ISMIP7")
- print("-" * 80)
- for idx, row in df.iterrows():
- run_greenland(
- config_file,
- template_file,
- path=path,
- resolution=resolution,
- nodes=nodes,
- ntasks=ntasks,
- queue=queue,
- walltime=walltime,
- debug=debug,
- uq=default,
- sample=int(row["sample"]) if "sample" in row else idx,
- )
+def _build_cli_parser(description: str, *, supports_execute: bool) -> ArgumentParser:
+ """
+ Build the argparse parser shared by ``run_forward`` and ``run_inverse``.
+ ``UQ_FILE`` is exposed as an *optional* positional: omit it to render one
+ job script (single mode), supply it to render an ensemble.
-def run_ensemble():
- """
- Run single glacier.
+ Parameters
+ ----------
+ description : str
+ Parser description shown in ``--help``.
+ supports_execute : bool
+ Whether to add the ``--execute`` flag. Currently a placeholder for
+ symmetry with the glacier CLI; ISMIP7 templates are normally
+ submitted via SLURM rather than executed in-process.
+
+ Returns
+ -------
+ argparse.ArgumentParser
+ Configured parser.
"""
-
- # set up the option parser
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
- parser.description = "Run RGI Glacier Ensemble."
+ parser.description = description
parser.add_argument(
"--output-path",
- help="Base path to save all files to. Files will be saved in `f'{out_path}/{RGI_ID}/output/'`.",
+ help="Base path to save all files to.",
type=str,
default="data",
)
parser.add_argument(
- "--queue",
- help="Overrides queue in config file.",
+ "--data-path",
+ help="Shared directory for staged input data (reused across runs). " "Defaults to /input.",
type=str,
default=None,
)
parser.add_argument(
- "--ntasks",
- help="Overrides ntatsks in config file.",
- type=int,
- default=None,
- )
- parser.add_argument(
- "--nodes",
- help="Overrides nodes in config file.",
- type=int,
- default=None,
+ "--force-overwrite",
+ help="Force downloading all files.",
+ action="store_true",
+ default=False,
)
+ parser.add_argument("--queue", type=str, default=None, help="Overrides queue in config file.")
+ parser.add_argument("--ntasks", type=int, default=None, help="Numbers of cores.")
+ parser.add_argument("--tasks", type=int, default=None, help="Cores per node.")
+ parser.add_argument("--nodes", type=int, default=None, help="Overrides nodes in config file.")
+ parser.add_argument("--walltime", type=str, default=None, help="Overrides walltime in config file.")
parser.add_argument(
- "--walltime",
- help="Overrides walltime in config file.",
- type=str,
- default=None,
+ "--resolution", type=_nullable_string, default=None, help="Override horizontal grid resolution."
)
parser.add_argument(
- "--resolution",
- help="Override horizontal grid resolution.",
- type=str,
+ "--stress-balance",
+ type=_nullable_string,
default=None,
+ help="Override the [stress_balance].model selection (e.g. 'sia', 'blatter').",
)
+ parser.add_argument("--start", type=_nullable_string, default=None, help="Override the time.start selection.")
+ parser.add_argument("--end", type=_nullable_string, default=None, help="Override the time.end selection.")
parser.add_argument(
"--posterior-file",
- help="CSV file posterior parameter distributions to sample from. Default=None.",
- type=str,
+ type=_nullable_string,
default=None,
+ help="CSV file of posterior parameter distributions to sample from (ensemble mode only).",
)
+ if supports_execute:
+ parser.add_argument(
+ "--execute",
+ action="store_true",
+ help="Reserved for parity with the glacier CLI; currently a no-op for ISMIP7.",
+ )
parser.add_argument(
"--debug",
- help="Debug or testing mode, do not write template, just the run command.",
action="store_true",
default=False,
+ help="Debug or testing mode, do not write template, just the run command.",
)
parser.add_argument(
- "--force-overwrite",
- help="Force downloading all files.",
- action="store_true",
- default=False,
- )
- parser.add_argument(
- "CONFIG_FILE",
- help="CONFIG TOML.",
- nargs=1,
- )
- parser.add_argument(
- "TEMPLATE_FILE",
- help="TEMPLATE J2.",
- nargs=1,
+ "--pism-config-cdl",
+ type=_nullable_string,
+ default=None,
+ help="Path to PISM CDL config file for option validation.",
)
+ parser.add_argument("CONFIG_FILE", help="CONFIG TOML.")
+ parser.add_argument("TEMPLATE_FILE", help="TEMPLATE J2.")
parser.add_argument(
"UQ_FILE",
- help="UQ TOML.",
- nargs=1,
+ nargs="?",
+ default=None,
+ type=_nullable_string,
+ help="UQ TOML (optional). Supply to render an ensemble; omit for a single run.",
)
+ return parser
- options, _ = parser.parse_known_args()
- force_overwrite = options.force_overwrite
- path = options.output_path
- config_file = options.CONFIG_FILE[0]
- template_file = options.TEMPLATE_FILE[0]
- uq_file = options.UQ_FILE[0]
- resolution = options.resolution
- posterior_file = options.posterior_file
- debug = options.debug
- queue = options.queue
- ntasks = options.ntasks
- nodes = options.nodes
- walltime = options.walltime
-
- path = Path(path)
- path.mkdir(parents=True, exist_ok=True)
- input_path = path / Path("input")
- input_path.mkdir(parents=True, exist_ok=True)
- output_path = path / Path("output")
- output_path.mkdir(parents=True, exist_ok=True)
- cfg = load_config(config_file)
- campaign_config = cfg.campaign.as_params()
-
- bucket = campaign_config["bucket"]
- prefix = campaign_config["prefix"]
+def _build_ensemble_df(
+ df: pd.DataFrame,
+ uq_file: Path,
+ output_path: Path,
+ posterior_file: str | Path | None,
+ seed: int = 42,
+) -> pd.DataFrame:
+ """
+ Build the per-member DataFrame for an ensemble run.
- df = stage(campaign_config, bucket=bucket, prefix=prefix, path=path, force_overwrite=force_overwrite)
+ Samples the UQ specification, optionally folds in a posterior CSV, then
+ cross-joins with the staged DataFrame ``df`` and assigns a composite
+ ``sample`` ID per row.
- default = {
- "input.file": df["boot_file"].iloc[0],
- "input.regrid.file": df["regrid_file"].iloc[0],
- "geometry.front_retreat.prescribed.file": df["retreat_file"].iloc[0],
- "grid.file": df["grid_file"].iloc[0],
- "atmosphere.given.file": df["climate_file"].iloc[0],
- "surface.given.file": df["climate_file"].iloc[0],
- "ocean.th.file": df["ocean_file"].iloc[0],
- }
-
- seed = 42
+ Parameters
+ ----------
+ df : pandas.DataFrame
+ Output of :func:`pism_terra.ismip7.greenland.stage.stage` (one row
+ per staged forcing tuple).
+ uq_file : Path
+ Path to the UQ TOML.
+ output_path : Path
+ Directory under which the realised sample CSV is persisted.
+ posterior_file : str or Path or None
+ Optional CSV of posterior parameter draws to override / extend the
+ UQ samples with.
+ seed : int, default 42
+ Seed for sampling (and posterior row choice).
+
+ Returns
+ -------
+ pandas.DataFrame
+ Per-ensemble-member DataFrame with all columns from ``df`` plus the
+ sampled UQ columns and a composite string ``sample`` column.
+ """
rng = np.random.default_rng(seed=seed)
uq = load_uq(uq_file)
n_samples = uq.samples
- mapping = uq.mapping
- uq_df = create_samples(uq.to_flat(), n_samples=n_samples, seed=seed)
+ uq_df = generate_samples(uq.to_flat(), n_samples=n_samples, method=uq.method, seed=seed)
+
if posterior_file is not None:
posterior_df = pd.read_csv(posterior_file).drop(columns=["Unnamed: 0", "exp_id"], errors="ignore")
choice_indices = rng.choice(range(len(posterior_df)), n_samples)
@@ -560,34 +950,196 @@ def run_ensemble():
uq_df = uq_df.drop(columns=duplicate_cols)
uq_df = pd.concat([uq_df, posterior_sampled_df], axis=1)
- uq_file = output_path / Path("uq.csv")
- uq_df.rename(columns={"sample": "id"}).to_csv(uq_file, index=False)
+ uq_df.rename(columns={"sample": "uq"}).to_csv(output_path / "uq.csv", index=False)
+
+ if uq.mapping:
+ uq_df = apply_choice_mapping(uq_df, df, uq.mapping)
+
+ merged_df = df.merge(uq_df, how="cross", suffixes=("_df", "_uq"))
+ merged_df["sample"] = merged_df["sample_df"].astype(str) + "_uq_" + merged_df["sample_uq"].astype(int).astype(str)
+ merged_df = merged_df.drop(columns=["sample_df", "sample_uq"])
+ return merged_df
+
+
+def _run(*, kind: str) -> None:
+ """
+ Shared CLI body for ISMIP7 Greenland forward and inverse runs.
+
+ Parses arguments, stages inputs, optionally builds an ensemble, then
+ renders one run script per member by calling ``_render__run``.
+ The two CLI entry points :func:`run_forward` and :func:`run_inverse`
+ are one-line wrappers around this function.
+
+ Parameters
+ ----------
+ kind : {"forward", "inverse"}
+ Which run script template to render. Selects the per-row worker
+ and decides whether to forward ``inverse.file`` to PISM.
+ """
+ if kind not in ("forward", "inverse"):
+ raise ValueError(f"kind must be 'forward' or 'inverse', got {kind!r}")
+ render = _render_forward_run if kind == "forward" else _render_inverse_run
+
+ parser = _build_cli_parser(
+ description=f"Stage ISMIP7 Greenland and render a {kind} run script (ensemble if UQ_FILE is given).",
+ supports_execute=False,
+ )
+ options = parser.parse_args()
+ force_overwrite = options.force_overwrite
+
+ path = Path(options.output_path)
+ path.mkdir(parents=True, exist_ok=True)
+ # Input data location is handled by ``stage`` (``--data-path`` or /input).
+ data_path = options.data_path
+ output_path = path / Path("output")
+ output_path.mkdir(parents=True, exist_ok=True)
+
+ config_file = options.CONFIG_FILE
+ template_file = options.TEMPLATE_FILE
+ uq_file = options.UQ_FILE
+ pism_config_cdl = options.pism_config_cdl
+
+ cfg = load_config(config_file)
+ campaign_config = cfg.campaign.as_params()
+
+ # Skip staging the (large) projection forcing when it isn't used:
+ # - inverse runs only use the historical forcing;
+ # - Historical experiments (C001/C002, product_leg == "historical") are
+ # historical-only and run no projection continuation.
+ counter = cfg.run_info.counter
+ historical_only = bool(counter and resolve_counter(counter).product_leg == "historical")
+ include_projection = kind == "forward" and not historical_only
+ df = stage(
+ campaign_config,
+ path=path,
+ force_overwrite=force_overwrite,
+ include_projection=include_projection,
+ data_path=data_path,
+ )
+
+ if uq_file is not None:
+ rows_df = _build_ensemble_df(df, uq_file, output_path, options.posterior_file)
+ header = f"Generate Ensemble {kind.capitalize()} Runs for ISMIP7 Greenland"
+ else:
+ rows_df = df
+ header = f"Generate {kind.capitalize()} Run for ISMIP7 Greenland"
+ is_ensemble = uq_file is not None
f = Figlet(font="standard")
banner = f.renderText("pism-terra")
- print("=" * 80)
+ print("=" * 120)
print(banner)
- print("=" * 80)
- print("Generate Ensemble Runs for Greenland")
- print("-" * 80)
- if uq.mapping:
- uq_df = apply_choice_mapping(uq_df, df, uq.mapping)
- for idx, row in uq_df.iterrows():
- run_greenland(
+ print("=" * 120)
+ print(header)
+ print("-" * 120)
+
+ config_cli = {
+ "resolution": options.resolution,
+ "nodes": options.nodes,
+ "ntasks": options.ntasks,
+ "tasks": options.tasks,
+ "queue": options.queue,
+ "walltime": options.walltime,
+ "stress_balance": options.stress_balance,
+ "start": options.start,
+ "end": options.end,
+ }
+
+ for idx, row in rows_df.iterrows():
+ if is_ensemble:
+ # Drop the staged columns and the composite sample id; whatever
+ # remains is a row of UQ overrides to forward to PISM.
+ uq_overrides = row.drop(labels=list(df.columns) + ["sample"]).to_dict()
+ else:
+ uq_overrides = {}
+
+ # File paths from the staging table override UQ-supplied paths for the
+ # same flag (matches the glacier behavior). ``uq_overrides`` carries
+ # the *historical*-epoch paths (which populate ``run_hist``); the
+ # projection paths ride on ``proj_overrides`` and only touch
+ # ``run_proj``.
+ uq_overrides.update(
+ {
+ "input.file": row["boot_file"],
+ "input.regrid.file": row["regrid_file"],
+ "grid.file": row["grid_file"],
+ "energy.bedrock_thermal.file": row["heatflux_file"],
+ "surface.ismip7.reference.file": row["boot_file"],
+ "atmosphere.given.file": row["climate_hist_file"],
+ "surface.given.file": row["climate_hist_file"],
+ "surface.ismip7.file": row["climate_hist_file"],
+ "surface.ismip7.gradient.file": row["climate_gradient_hist_file"],
+ "ocean.pico.file": row["ocean_hist_file"],
+ "ocean.picop.file": row["ocean_hist_file"],
+ "ocean.th.file": row["ocean_hist_file"],
+ "frontal_melt.routing.file": row["ocean_hist_file"],
+ }
+ )
+ # Projection-epoch overrides only exist when the projection forcing was
+ # staged. The inverse run and historical-only experiments (C001/C002)
+ # skip it, so the ``*_proj_file`` columns are absent from the row (see
+ # ``include_projection`` above / stage(..., include_projection=...)).
+ proj_overrides = None
+ if include_projection:
+ proj_overrides = {
+ "atmosphere.given.file": row["climate_proj_file"],
+ "surface.given.file": row["climate_proj_file"],
+ "surface.ismip7.file": row["climate_proj_file"],
+ "surface.ismip7.gradient.file": row["climate_gradient_proj_file"],
+ "ocean.pico.file": row["ocean_proj_file"],
+ "ocean.picop.file": row["ocean_proj_file"],
+ "ocean.th.file": row["ocean_proj_file"],
+ "frontal_melt.routing.file": row["ocean_proj_file"],
+ }
+ # Wire the inverse observation file only when the stage produced one
+ # (campaign config can opt in via an ``obs_file`` key); otherwise
+ # rely on whatever ``inverse.file`` the UQ supplied.
+ if kind == "inverse" and "obs_file" in row and pd.notna(row["obs_file"]):
+ uq_overrides["inverse.file"] = row["obs_file"]
+
+ outline_file = row["outline_file"] if "outline_file" in row else None
+ # ISMIP7 staging uses the GCM name (a string like "CESM2-WACCM") as
+ # the sample id, so don't try to coerce to int the way the glacier
+ # CLI does.
+ sample = row["sample"] if "sample" in row else idx
+ render(
config_file,
template_file,
+ outline_file,
path=path,
- resolution=resolution,
- nodes=nodes,
- ntasks=ntasks,
- queue=queue,
- walltime=walltime,
- debug=debug,
- uq=default,
- sample=int(row["sample"]) if "sample" in row else idx,
+ config_cli=config_cli,
+ debug=options.debug,
+ uq=uq_overrides,
+ sample=sample,
+ pism_config_cdl=pism_config_cdl,
+ proj_overrides=proj_overrides,
)
+def run_forward() -> None:
+ """
+ CLI entry point for ISMIP7 Greenland forward runs (single or ensemble).
+
+ Behaves as a single run when no ``UQ_FILE`` positional is supplied, and
+ as a UQ ensemble when one is. The argument schema and output layout are
+ otherwise identical.
+ """
+ _run(kind="forward")
+
+
+def run_inverse() -> None:
+ """
+ CLI entry point for ISMIP7 Greenland inverse runs (single or ensemble).
+
+ Behaves as a single inverse run when no ``UQ_FILE`` positional is
+ supplied, and as a UQ ensemble when one is. When the staged row
+ includes an ``obs_file`` column (set by the campaign config), it is
+ wired through as ``inverse.file``; otherwise the user can pass
+ ``inverse.file`` via the UQ TOML.
+ """
+ _run(kind="inverse")
+
+
if __name__ == "__main__":
__spec__ = None # type: ignore
- run_single()
+ run_forward()
diff --git a/pism_terra/ismip7/greenland/stage.py b/pism_terra/ismip7/greenland/stage.py
index b45b8b7..28e8f99 100644
--- a/pism_terra/ismip7/greenland/stage.py
+++ b/pism_terra/ismip7/greenland/stage.py
@@ -24,7 +24,7 @@
import time
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from collections.abc import Mapping
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Callable
@@ -35,8 +35,9 @@
import xarray as xr
from pyfiglet import Figlet
from shapely.geometry import Polygon
+from tqdm.auto import tqdm
-from pism_terra.aws import local_to_s3, s3_to_local
+from pism_terra.aws import download_from_s3, local_to_s3
from pism_terra.config import load_config
from pism_terra.workflow import check_dataset_fully, check_xr_fully, check_xr_lazy
@@ -46,9 +47,9 @@
def stage(
config: dict,
path: str | Path = "input_files",
- bucket: str = "pism-cloud-data",
- prefix: str = "ismip7_greenland_input",
force_overwrite: bool = False,
+ include_projection: bool = True,
+ data_path: str | Path | None = None,
) -> pd.DataFrame:
"""
Stage ISMIP7 Greenland inputs and return a file index.
@@ -69,77 +70,178 @@ def stage(
Path to the heatflux NetCDF file relative to the input directory.
- ``"regrid_file"`` : str
Path to the regrid NetCDF file relative to the input directory.
- - ``"retreat_file"`` : str
- Path to the retreat NetCDF file relative to the input directory.
- - ``"pathway"`` : str
- ISMIP7 pathway identifier.
- - ``"gcm"`` : str
- GCM model name.
+ - ``"gcms"`` : str or list[str]
+ GCM model name(s).
- ``"version"`` : str
Dataset version.
- - ``"start_year"`` : int
- Start year of the forcing period.
- - ``"end_year"`` : int
- End year of the forcing period.
+ - ``"historical_start_year"`` : int
+ First year of the historical forcing file.
+ - ``"historical_end_year"`` : int
+ Last (inclusive) year of the historical forcing file.
+
+ The following are required only when ``include_projection`` is ``True``
+ (forward runs); an inverse/calibration config may omit them:
+
+ - ``"pathway"`` : str
+ ISMIP7 pathway identifier.
+ - ``"projection_start_year"`` : int
+ First year of the projection forcing file.
+ - ``"projection_end_year"`` : int
+ Last (inclusive) year of the projection forcing file
+ (differs per pathway — 2100 for ssp370, 2300 for ssp126/585).
path : str or pathlib.Path, default ``"input_files"``
Output directory. Created if missing. All staged artifacts are written here.
- bucket : str, default ``"pism-cloud-data"``
- AWS S3 bucket name to sync ISMIP7 input data from.
- prefix : str, default ``"ismip7_greenland_input"``
- S3 key prefix (folder path within the bucket).
force_overwrite : bool, default ``False``
If ``True``, downstream helpers may regenerate intermediate/final artifacts
even if cache files exist.
+ include_projection : bool, default ``True``
+ If ``True``, stage both the historical and the projection (pathway) forcing
+ epochs. If ``False``, stage only the historical epoch and omit the ``*_proj_*``
+ columns from the returned index. The inverse run passes ``False`` because it
+ only uses the historical forcing, so the (large) projection files never need
+ to be downloaded.
+ data_path : str or pathlib.Path or None, default ``None``
+ Directory where the staged input data is written. When given, all inputs go
+ here (a shared location that multiple experiment output directories can
+ reuse to save disk); when ``None``, they go to ``/input`` as before.
+ Files already present are not re-downloaded, so pointing several runs at the
+ same ``data_path`` stages the data once.
Returns
-------
pandas.DataFrame
- Single-row DataFrame with absolute-path columns including
+ DataFrame with one row per GCM and absolute-path columns including
``boot_file``, ``grid_file``, ``heatflux_file``, ``regrid_file``,
- ``retreat_file``, ``climate_file``, ``ocean_file``,
- ``surface_input_file``, and ``frontal_melt_file``.
+ ``outline_file``, ``climate_file``, ``ocean_file``,
+ ``surface_input_file``, ``frontal_melt_file``, and ``sample``.
"""
f = Figlet(font="standard")
banner = f.renderText("pism-terra")
- print("=" * 80)
+ print("=" * 120)
print(banner)
- print("=" * 80)
+ print("=" * 120)
print("Stage ISMIP7 Greenland")
- print("-" * 80)
+ print("-" * 120)
print("")
# Outputs dir
path = Path(path)
path.mkdir(parents=True, exist_ok=True)
- input_path = path / Path("input")
+ # Input data goes to a shared ``data_path`` when given (so several experiment
+ # output dirs can reuse one staged copy), otherwise to ``/input``.
+ input_path = Path(data_path) if data_path is not None else path / Path("input")
if force_overwrite:
input_path.unlink(missing_ok=True)
input_path.mkdir(parents=True, exist_ok=True)
- s3_to_local(bucket, prefix=prefix, dest=input_path)
+ bucket = config["bucket"]
+ # ``prefix`` is a plain string in CampaignConfig; build the S3-side
+ # prefix with f-string concatenation rather than Path division (Path /
+ # str would TypeError on the leading str, and S3 keys aren't filesystem
+ # paths anyway). Include ``version`` so the URL resolves to the layout
+ # ``prepare`` writes, e.g. ``s3://…/ismip7/greenland/input/v2/…``.
+ prefix = f"{config['prefix']}/{config['version']}"
- grid_file = input_path / Path(config["grid_file"])
- check_xr_fully(grid_file)
+ gcms = config["gcms"]
+ gcms = [gcms] if isinstance(gcms, str) else gcms
+ version = config["version"]
+ # Historical and projection year ranges. End years are inclusive per
+ # the campaign-config convention (see CampaignConfig). Projection years are
+ # only needed when the projection epoch is staged (forward runs).
+ hist_start = int(config["historical_start_year"])
+ hist_end = int(config["historical_end_year"])
+ grid_file = input_path / Path(config["grid_file"])
boot_file = input_path / Path(config["boot_file"])
- check_xr_lazy(boot_file)
-
heatflux_file = input_path / Path(config["heatflux_file"])
- check_xr_lazy(heatflux_file)
-
regrid_file = input_path / Path(config["regrid_file"])
- check_xr_lazy(regrid_file)
-
- retreat_file = input_path / Path(config["retreat_file"])
- check_xr_lazy(retreat_file)
+ outline_file = input_path / Path(config["outline_file"])
+ obs_file = input_path / Path(config["obs_file"])
+
+ # Enumerate every S3 key we actually need so we don't bulk-sync the
+ # whole prefix (which carries per-GCM forcings for GCMs we aren't
+ # running, plus assorted bookkeeping files). ``required_files`` pairs
+ # the rel-key under ``prefix`` with its target local path.
+ required_files: list[tuple[str, Path]] = [
+ (config["grid_file"], grid_file),
+ (config["boot_file"], boot_file),
+ (config["heatflux_file"], heatflux_file),
+ (config["regrid_file"], regrid_file),
+ (config["outline_file"], outline_file),
+ (config["obs_file"], obs_file),
+ ]
+ # ``prepare`` publishes one file per (epoch, gcm, forcing) — the
+ # historical span and each projection pathway live side-by-side.
+ # ``climate_gradient`` is the annual elevation-gradient companion of
+ # ``climate`` (see ``prepare_ismip7_forcing``); validation downstream
+ # expects all three per epoch.
+ #
+ # Per-epoch spec: (pathway_name, start_year, end_year, column_suffix)
+ epoch_specs = [
+ ("historical", hist_start, hist_end, "hist"),
+ ]
+ if include_projection:
+ # ``pathway`` and the projection year range are only needed for the
+ # projection epoch, so an inverse/calibration config need not set them.
+ pathway = config["pathway"]
+ proj_start = int(config["projection_start_year"])
+ proj_end = int(config["projection_end_year"])
+ epoch_specs.append((pathway, proj_start, proj_end, "proj"))
+ for gcm in gcms:
+ for epoch_pathway, ep_start, ep_end, _ in epoch_specs:
+ for forcing in ("climate", "climate_gradient", "ocean"):
+ rel = f"ismip7_greenland_{forcing}_{epoch_pathway}_{gcm}_{version}_{ep_start}_{ep_end}.nc"
+ required_files.append((rel, input_path / rel))
+
+ # Skip files that already exist locally unless force_overwrite is set.
+ # We intentionally do not re-validate cached files here; the explicit
+ # validation passes below do that and surface failures.
+ to_download = [
+ (rel_key, local_path) for rel_key, local_path in required_files if force_overwrite or not local_path.exists()
+ ]
+
+ if to_download:
+ # boto3 clients are safe for concurrent ``download_from_s3``; the
+ # outer 4-way fan-out keeps the connection pool busy without
+ # interleaving the per-file tqdm bars too aggressively.
+ # Use a distinct name from the ``ProcessPoolExecutor`` blocks below
+ # so mypy doesn't narrow the variable's type across reuse.
+ with ThreadPoolExecutor(max_workers=4) as dl_executor:
+ futures = {
+ dl_executor.submit(
+ download_from_s3,
+ f"s3://{bucket}/{prefix}/{rel_key}",
+ local_path,
+ ): rel_key
+ for rel_key, local_path in to_download
+ }
+ for dl_future in as_completed(futures):
+ try:
+ dl_future.result()
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ print(f"Failed to download s3://{bucket}/{prefix}/{futures[dl_future]}: {exc}")
+
+ # Grid file gets the heavier full-load check; leave it sequential.
+ check_xr_fully(grid_file)
- pathway = config["pathway"]
- gcm = config["gcm"]
- version = config["version"]
- start_year = config["start_year"]
- end_year = config["end_year"]
+ # Validate the lazy-check inputs concurrently; only invalid files print.
+ input_lazy_files = [boot_file, heatflux_file, regrid_file]
+ # Processes (not threads): HDF5 isn't reliably thread-safe across all
+ # builds (Chinook segfaults), so each worker gets its own interpreter
+ # and HDF5 state.
+ with ProcessPoolExecutor(max_workers=8) as executor:
+ future_to_path = {executor.submit(check_xr_lazy, p, verbose=False): p for p in input_lazy_files}
+ for future in tqdm(
+ as_completed(future_to_path),
+ total=len(future_to_path),
+ desc="Checking input files",
+ unit="file",
+ ):
+ p = future_to_path[future]
+ if not future.result():
+ print(f"{p.resolve()} is not valid ✗")
# Build file index (one row per climate file)
files_dict = {
@@ -147,31 +249,73 @@ def stage(
"grid_file": grid_file.resolve(),
"heatflux_file": heatflux_file.resolve(),
"regrid_file": regrid_file.resolve(),
- "retreat_file": retreat_file.resolve(),
+ "outline_file": outline_file.resolve(),
+ "obs_file": obs_file.resolve(),
}
- for forcing in ["climate", "ocean"]:
- forcing_file = input_path / Path(
- f"ismip7_greenland_{forcing}_{pathway}_{gcm}_v{version}_{start_year}_{end_year}.nc"
- )
- check_xr_lazy(forcing_file)
- files_dict[f"{forcing}_file"] = forcing_file.resolve()
-
- forcing = "climate"
- surface_input_file = input_path / Path(
- f"ismip7_greenland_{forcing}_{pathway}_{gcm}_v{version}_{start_year}_{end_year}.nc"
- )
- check_xr_lazy(surface_input_file)
- files_dict["surface_input_file"] = surface_input_file.resolve()
- forcing = "ocean"
- frontal_melt_file = input_path / Path(
- f"ismip7_greenland_{forcing}_{pathway}_{gcm}_v{version}_{start_year}_{end_year}.nc"
- )
- check_xr_lazy(frontal_melt_file)
- files_dict["frontal_melt_file"] = frontal_melt_file.resolve()
+ # Per-GCM per-epoch forcing paths. Every forward run needs both a
+ # historical file and a projection file per (climate, climate_gradient,
+ # ocean); ``run.py`` wires them into ``run_hist`` and ``run_proj``
+ # respectively. Aliases keep the column names PISM's ISMIP6 module
+ # expects (``surface_input`` = climate, ``frontal_melt`` = ocean).
+ def _forcing_path(forcing: str, gcm: str, epoch_pathway: str, ep_start: int, ep_end: int) -> Path:
+ """
+ Build the local path for one (epoch, forcing, gcm) NetCDF.
+
+ Parameters
+ ----------
+ forcing : str
+ ``"climate"``, ``"climate_gradient"``, or ``"ocean"``.
+ gcm : str
+ GCM name (e.g. ``"CESM2-WACCM"``).
+ epoch_pathway : str
+ ``"historical"`` or the projection pathway (e.g. ``"ssp370"``).
+ ep_start, ep_end : int
+ Inclusive year range embedded in the filename.
+
+ Returns
+ -------
+ pathlib.Path
+ Absolute path to the forcing file under ``input_path``.
+ """
+ return input_path / Path(f"ismip7_greenland_{forcing}_{epoch_pathway}_{gcm}_{version}_{ep_start}_{ep_end}.nc")
+
+ forcing_paths: dict[tuple[str, str, str], Path] = {}
+ for gcm in gcms:
+ for epoch_pathway, ep_start, ep_end, epoch_key in epoch_specs:
+ for forcing in ("climate", "climate_gradient", "ocean"):
+ forcing_paths[(gcm, epoch_key, forcing)] = _forcing_path(forcing, gcm, epoch_pathway, ep_start, ep_end)
+
+ # Processes (not threads): HDF5 isn't reliably thread-safe across all
+ # builds (Chinook segfaults), so each worker gets its own interpreter
+ # and HDF5 state.
+ with ProcessPoolExecutor(max_workers=8) as executor:
+ future_to_path = {executor.submit(check_xr_lazy, p, verbose=False): p for p in forcing_paths.values()}
+ for future in tqdm(
+ as_completed(future_to_path),
+ total=len(future_to_path),
+ desc="Checking forcing files",
+ unit="file",
+ ):
+ p = future_to_path[future]
+ if not future.result():
+ print(f"{p.resolve()} is not valid ✗")
dfs: list[pd.DataFrame] = []
- dfs.append(pd.DataFrame.from_dict([files_dict]))
+ epoch_keys = [key for *_, key in epoch_specs]
+ for gcm in gcms:
+ row = dict(files_dict)
+ for epoch_key in epoch_keys:
+ climate_f = forcing_paths[(gcm, epoch_key, "climate")]
+ climate_grad_f = forcing_paths[(gcm, epoch_key, "climate_gradient")]
+ ocean_f = forcing_paths[(gcm, epoch_key, "ocean")]
+ row[f"climate_{epoch_key}_file"] = climate_f.resolve()
+ row[f"climate_gradient_{epoch_key}_file"] = climate_grad_f.resolve()
+ row[f"ocean_{epoch_key}_file"] = ocean_f.resolve()
+ row[f"surface_input_{epoch_key}_file"] = climate_f.resolve()
+ row[f"frontal_melt_{epoch_key}_file"] = ocean_f.resolve()
+ row["sample"] = gcm
+ dfs.append(pd.DataFrame.from_dict([row]))
df = pd.concat(dfs).reset_index(drop=True)
return df
@@ -197,6 +341,12 @@ def main():
type=Path,
default=Path("data/ismip7_greenland"),
)
+ parser.add_argument(
+ "--data-path",
+ help="Shared directory for staged input data (reused across runs). " "Defaults to /input.",
+ type=Path,
+ default=None,
+ )
parser.add_argument(
"--force-overwrite",
help="Force downloading all files.",
@@ -211,6 +361,7 @@ def main():
options, unknown = parser.parse_known_args()
path = options.output_path
+ data_path = options.data_path
config_file = options.CONFIG_FILE[0]
force_overwrite = options.force_overwrite
@@ -219,8 +370,9 @@ def main():
path.mkdir(parents=True, exist_ok=True)
- is_df = stage(config, path=path, force_overwrite=force_overwrite)
- is_df.to_csv(path / Path("input") / Path("ismip7_greenland_files.csv"))
+ is_df = stage(config, path=path, force_overwrite=force_overwrite, data_path=data_path)
+ input_dir = Path(data_path) if data_path is not None else path / Path("input")
+ is_df.to_csv(input_dir / Path("ismip7_greenland_files.csv"))
if options.bucket:
prefix = f"{options.bucket_prefix}/ismip7_greenland" if options.bucket_prefix else "ismip7_greenland"
diff --git a/pism_terra/ismip7/naming.py b/pism_terra/ismip7/naming.py
new file mode 100644
index 0000000..8d6cfe2
--- /dev/null
+++ b/pism_terra/ismip7/naming.py
@@ -0,0 +1,182 @@
+# Copyright (C) 2026 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+"""
+ISMIP7 submission directory and file naming (see the conventions doc, section 8).
+
+Directory structure::
+
+ /////
+
+File name::
+
+ _____
+ ____.nc
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+# First letter of ``set_counter`` for each ISMIP7 set type.
+_SET_LETTER = {"CORE": "C", "ESM": "E", "PPE": "P"}
+
+
+def member_ids(set_id: str, sample: int) -> tuple[str, str, str]:
+ """
+ Derive ``(set_counter, ISM_member_id, forcing_member_id)`` from a sample index.
+
+ The ``set_counter`` always increments with the 0-based ``sample`` (it is a
+ unique index within the set). Which member id varies depends on the set type
+ (see the ISMIP7 conventions, section 8):
+
+ * ``CORE`` -- both members fixed at ``m001`` / ``f001``.
+ * ``PPE`` -- ICE member ``mNNN`` varies (perturbed ICE physics), forcing
+ fixed at ``f001``.
+ * ``ESM`` -- forcing member ``fNNN`` varies (different ESM/forcing), ICE
+ fixed at ``m001``.
+
+ A single (non-ensemble) run yields the first counter, e.g.
+ ``("C001", "m001", "f001")`` for a CORE set.
+
+ Parameters
+ ----------
+ set_id : str
+ ISMIP7 set type, one of ``"CORE"``, ``"ESM"``, ``"PPE"``.
+ sample : int
+ 0-based ensemble member index.
+
+ Returns
+ -------
+ tuple of str
+ ``(set_counter, ism_member_id, forcing_member_id)``, e.g.
+ ``("C001", "m001", "f001")``.
+
+ Raises
+ ------
+ ValueError
+ If ``set_id`` is not a recognized ISMIP7 set type.
+ """
+ key = set_id.upper()
+ try:
+ letter = _SET_LETTER[key]
+ except KeyError as exc:
+ raise ValueError(f"set_id must be one of {sorted(_SET_LETTER)}, got {set_id!r}") from exc
+ n = int(sample) + 1
+ set_counter = f"{letter}{n:03d}"
+ ism_member = f"m{n:03d}" if key == "PPE" else "m001"
+ forcing_member = f"f{n:03d}" if key == "ESM" else "f001"
+ return set_counter, ism_member, forcing_member
+
+
+@dataclass(frozen=True)
+class ISMIP7Names: # pylint: disable=too-many-instance-attributes
+ """
+ Resolved ISMIP7 naming components for one model run/member.
+
+ Attributes
+ ----------
+ domain_id : str
+ ``"GrIS"`` or ``"AIS"``.
+ source_id : str
+ Modelling group (no underscores/dots/special characters).
+ ism_id : str
+ Ice-sheet model name and version (no underscores/dots/special characters).
+ ism_member_id : str
+ ICE member variant, ``mNNN``.
+ esm_id : str
+ Forcing CMIP/ESM model name, e.g. ``"CESM2-WACCM"``.
+ forcing_member_id : str
+ Forcing variant, ``fNNN``.
+ experiment_id : str
+ e.g. ``"historical"``, ``"ssp370"``, ``"ctrl"``.
+ set_id : str
+ ``"CORE"``/``"ESM"``/``"PPE"``.
+ set_counter : str
+ Unique index within the set, ``CNNN``/``ENNN``/``PNNN``.
+ time_range : str
+ Year or year range, e.g. ``"1980-2014"``.
+ """
+
+ domain_id: str
+ source_id: str
+ ism_id: str
+ ism_member_id: str
+ esm_id: str
+ forcing_member_id: str
+ experiment_id: str
+ set_id: str
+ set_counter: str
+ time_range: str
+
+ def directory(self, root: Path | str = ".") -> Path:
+ """
+ Return the ISMIP7 submission directory for this run.
+
+ Parameters
+ ----------
+ root : pathlib.Path or str, default ``"."``
+ Base directory the ISMIP7 tree is created under.
+
+ Returns
+ -------
+ pathlib.Path
+ ``/////``.
+ """
+ return Path(root) / self.domain_id / self.source_id / self.ism_id / self.set_id / self.set_counter
+
+ def stem(self) -> str:
+ """
+ Return the filename stem shared by all variables of this run.
+
+ Returns
+ -------
+ str
+ Everything after ``_`` in the file name, i.e.
+ ``______
+ __``.
+ """
+ return "_".join(
+ (
+ self.domain_id,
+ self.source_id,
+ self.ism_id,
+ self.ism_member_id,
+ self.esm_id,
+ self.forcing_member_id,
+ self.experiment_id,
+ self.set_counter,
+ self.time_range,
+ )
+ )
+
+ def filename(self, variable_id: str) -> str:
+ """
+ Return the full ISMIP7 file name for one variable.
+
+ Parameters
+ ----------
+ variable_id : str
+ Variable short name (as defined in the ISMIP7 data request).
+
+ Returns
+ -------
+ str
+ ``_.nc``.
+ """
+ return f"{variable_id}_{self.stem()}.nc"
diff --git a/pism_terra/kitp/analyze.py b/pism_terra/kitp/analyze.py
new file mode 100644
index 0000000..8b4b4ac
--- /dev/null
+++ b/pism_terra/kitp/analyze.py
@@ -0,0 +1,719 @@
+# Copyright (C) 2026 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+# pylint: disable=unused-import,unused-variable
+
+"""
+Postprocessing.
+"""
+
+import logging
+import re
+import warnings
+from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+from collections.abc import Sequence
+from functools import partial
+from pathlib import Path
+from typing import Any
+
+import cftime
+import dask
+import matplotlib as mpl
+import matplotlib.pylab as plt
+import nc_time_axis
+import numpy as np
+import pint_xarray
+import seaborn as sns
+import xarray as xr
+from cmap import Colormap
+from cycler import cycler
+from dask.diagnostics import ProgressBar
+from matplotlib.lines import Line2D
+from matplotlib.patches import Patch
+
+from pism_terra.processing import preprocess_netcdf as preprocess
+
+cm = Colormap("tol:bright").to_matplotlib()
+cm_cycler = cycler(color=cm.colors)
+cm_precip = Colormap("crameri:navia").to_matplotlib()
+cm_rdbu = Colormap("crameri:vik_r").to_matplotlib()
+
+xr.set_options(keep_attrs=True)
+warnings.filterwarnings("ignore", message="Increasing number of chunks")
+warnings.filterwarnings("ignore", message="invalid value encountered in cast", category=RuntimeWarning)
+warnings.filterwarnings("ignore", message="pkg_resources is deprecated", category=UserWarning)
+
+logger = logging.getLogger("pism_terra.kitp.analyze")
+
+
+fontsize = 6
+rc_params = {
+ "axes.linewidth": 0.15,
+ "xtick.major.size": 2.0,
+ "xtick.major.width": 0.15,
+ "ytick.major.size": 2.0,
+ "ytick.major.width": 0.15,
+ "hatch.linewidth": 0.15,
+ "font.size": fontsize,
+ "font.family": "DejaVu Sans",
+}
+
+single_model_gcms = ["CESM1-WACCM-SC"]
+multi_model_gcms = [
+ "CESM1-WACCM-SC",
+ "AWI-CM-1-1-MR",
+ "CESM2",
+ "CNRM-CM6-1",
+ "CanESM5",
+ "HadGEM3-GC31-MM",
+ "IPSL-CM6A-LR",
+]
+
+BASELINE_OPTS = {"short_hand": "baseline", "color": (0, 0, 0), "ls": "dashed", "title": "Baseline"}
+
+EXPS_OPTS: dict[str, dict[str, Any]] = {
+ "pdSST-futArcSICSIT_pdSST-pdSICSIT": {
+ "color": (0.0660, 0.4430, 0.7450),
+ "ls": "solid",
+ "title": "Arctic sea ice loss (AGCM)",
+ },
+ "pa-futArcSIC-ext_pa-pdSIC-ext": {
+ "color": (0.8660, 0.3290, 0),
+ "ls": "solid",
+ "title": "Arctic sea ice loss (AOGCM)",
+ },
+ "futSST-pdSIC_pdSST-pdSIC": {
+ "color": (0.9290, 0.6940, 0.1250),
+ "ls": "solid",
+ "title": "Global SST warming",
+ },
+ "pdSST-futArcSIC_pdSST-pdSIC": {
+ "color": (0.5210, 0.0860, 0.8190),
+ "ls": "solid",
+ "title": "Arctic sea ice loss (AGCM + 2m ice)",
+ },
+ "futSST-futArcSIC-SUM_pdSST-pdSIC": {
+ "color": (0.2310, 0.6660, 0.1960),
+ "ls": "solid",
+ "title": "Global SST warming + SIC loss (AGCM + 2m ice)",
+ },
+}
+
+
+def load_dataset(filename_or_obj: Sequence[str | Path], join: str | None = "outer", **kwargs) -> xr.Dataset:
+ """
+ Load and preprocess multiple NetCDF files into a single dataset.
+
+ Parameters
+ ----------
+ filename_or_obj : list of str or Path
+ NetCDF files to open.
+ join : str, optional
+ How to combine datasets along any shared dimensions, by default "outer".
+ **kwargs
+ Forwarded to :func:`preprocess_netcdf`.
+
+ Returns
+ -------
+ xr.Dataset
+ The merged dataset.
+ """
+ time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
+ delta_coder = xr.coders.CFTimedeltaCoder()
+
+ kwargs.setdefault("process_config", False)
+ with ProgressBar():
+ _dss: list[xr.Dataset] = []
+ for f in filename_or_obj:
+ _ds = xr.open_dataset(
+ f,
+ engine="netcdf4",
+ decode_times=time_coder,
+ decode_timedelta=delta_coder,
+ )
+ if _ds.time.size > 0:
+ _ds = preprocess(_ds, **kwargs)
+ _dss.append(_ds)
+ ds = xr.concat(_dss, dim="time", join=join).sortby("time")
+ return ds
+
+
+def plot_scalar_timeseries(infiles: list[str | Path]):
+ """
+ Plot ice-mass change timeseries from scalar output files.
+
+ Parameters
+ ----------
+ infiles : list of str or Path
+ Scalar NetCDF files (must include one baseline HIRHAM5 file).
+ """
+
+ time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
+ delta_coder = xr.coders.CFTimedeltaCoder()
+
+ gt2mmsle = xr.DataArray(-1 / 361.8).pint.quantify("mm/Gt")
+ pctls = [0.05, 0.5, 0.95]
+ cumulative_vars = ["ice_mass"]
+ flux_vars = [
+ "tendency_of_ice_mass",
+ "tendency_of_ice_mass_due_to_surface_mass_flux",
+ "tendency_of_ice_mass_due_to_discharge",
+ ]
+
+ baseline_file = next(f for f in infiles if "HIRHAM5" in Path(f).name)
+ baseline = xr.open_dataset(baseline_file, chunks=None, decode_times=time_coder, decode_timedelta=delta_coder)
+ pism_config = baseline.pism_config
+ res = f"""{int(pism_config.attrs["grid.dx"])}m"""
+ if "basin" not in baseline.dims:
+ baseline = baseline.expand_dims({"basin": ["GIS"]})
+ baseline = baseline.resample({"time": "YE"}).mean()
+ baseline = baseline.assign_coords(time=[t.replace(year=t.year - 1) for t in baseline.time.values])
+ baseline = baseline.pint.quantify()
+
+ exp_files = [Path(f) for f in infiles if "HIRHAM5" not in Path(f).name]
+
+ # Group files by experiment, then classify experiments as single- or multi-GCM
+ exp_regexp = re.compile(r"_exp_((?:futSST|pdSST|pa)-\S+?)_(?:uq_\d+_)?\d{4}-\d{2}-\d{2}")
+ gcm_regexp = re.compile(r"_gcm_(.+?)_exp_")
+ files_by_exp: dict[str, list[Path]] = {}
+ for f in exp_files:
+ m = exp_regexp.search(f.name)
+ if m:
+ files_by_exp.setdefault(m.group(1), []).append(f)
+
+ single_model_files = []
+ multi_model_files = []
+ for exp_name, files in files_by_exp.items():
+ gcms = {m.group(1) for f in files if (m := gcm_regexp.search(f.name)) is not None}
+ if gcms - set(single_model_gcms):
+ multi_model_files.extend(files)
+ else:
+ single_model_files.extend(files)
+ logger.info(" %s: %d GCMs -> %s", exp_name, len(gcms), "multi" if gcms - set(single_model_gcms) else "single")
+
+ load_kwargs: dict[str, Any] = {
+ "exp_regexp": r"_exp_((?:futSST|pdSST|pa)-\S+?)_(?:uq_\d+_)?\d{4}-\d{2}-\d{2}",
+ "uq_regexp": None,
+ "uq_dim": None,
+ "exp_dim": "exp_id",
+ }
+
+ def _prepare(ds: xr.Dataset) -> xr.Dataset:
+ """
+ Drop object-dtype vars, ensure ``basin`` dim, resample to yearly means, and quantify.
+
+ Parameters
+ ----------
+ ds : xr.Dataset
+ Input dataset.
+
+ Returns
+ -------
+ xr.Dataset
+ Prepared dataset with yearly-mean values and pint-quantified variables.
+ """
+ obj_vars = [v for v in ds.data_vars if ds[v].dtype == object]
+ ds = ds.drop_vars(obj_vars)
+ if "basin" not in ds.dims:
+ ds = ds.expand_dims({"basin": ["GIS"]})
+ ds = ds.resample({"time": "YE"}).mean()
+ # Shift time coordinate by -1 year so the series starts at year 0
+ ds = ds.assign_coords(time=[t.replace(year=t.year - 1) for t in ds.time.values])
+ return ds.pint.quantify()
+
+ model_files = single_model_files + multi_model_files
+ logger.info("Loading GCM experiments")
+ with dask.config.set(**{"array.slicing.split_large_chunks": False}):
+ _dss: list[xr.Dataset] = []
+ gcm = _prepare(load_dataset(model_files, **load_kwargs))
+
+ baseline_cumulative_computed = baseline[cumulative_vars].pint.to("Gt").pint.dequantify().compute()
+ baseline_fluxes_computed = baseline[flux_vars].pint.to("Gt/yr").pint.dequantify().compute()
+ baseline_computed = xr.merge([baseline_cumulative_computed, baseline_fluxes_computed]).pint.quantify()
+
+ gcm_cumulative_computed = gcm[cumulative_vars].pint.to("Gt").pint.dequantify().compute()
+ gcm_fluxes_computed = gcm[flux_vars].pint.to("Gt/yr").pint.dequantify().compute()
+ gcm_computed = xr.merge([gcm_cumulative_computed, gcm_fluxes_computed]).pint.quantify()
+
+ gcm_sub_baseline = gcm_computed - baseline_computed
+
+ with mpl.rc_context(rc=rc_params):
+ for basin_name in baseline.basin.values:
+ basin_gcm = gcm_computed.sel(basin=basin_name)
+ basin_slc = ((basin_gcm["ice_mass"] - basin_gcm["ice_mass"].isel({"time": 0})) * gt2mmsle).pint.dequantify()
+ time_vals = basin_slc.time.values
+ basin_baseline = baseline_computed.sel(basin=basin_name)
+ basin_baseline_slc = (
+ (basin_baseline["ice_mass"] - basin_baseline["ice_mass"].isel({"time": 0})) * gt2mmsle
+ ).pint.dequantify()
+
+ fig, ax = plt.subplots(1, 1, figsize=(6.4, 3.6))
+
+ l = []
+
+ _l = basin_baseline_slc.plot(
+ ax=ax, color=BASELINE_OPTS["color"], ls=BASELINE_OPTS["ls"], label=BASELINE_OPTS["title"], lw=1
+ )
+ l.append(_l)
+
+ for exp_name, exp in EXPS_OPTS.items():
+ _gcm_slc = basin_slc.sel({"exp_id": exp_name})
+ _ = _gcm_slc.plot(ax=ax, hue="gcm_id", color=exp["color"], ls=exp["ls"], lw=0.75, add_legend=False)
+ _l = _gcm_slc.isel({"gcm_id": 0}).plot(
+ ax=ax, color=exp["color"], ls=exp["ls"], label=exp["title"], lw=0.75, add_legend=False
+ )
+ l.append(_l)
+
+ ax.axhline(y=0, color="k", ls="dotted", lw=0.5)
+ ax.set_ylabel("Contribution to sea-level (mm)")
+ ax.set_xlabel("Time")
+ ax.set_title(basin_name)
+ ax.set_xlim(time_vals[0], time_vals[-1])
+ year_ticks = [t for t in time_vals if t.year % 50 == 0]
+ ax.set_xticks(year_ticks)
+ ax.set_xticklabels([f"{int(t.year)}" for t in year_ticks])
+
+ leg_line = fig.legend(
+ handles=[h for item in l for h in (item if isinstance(item, list) else [item])],
+ loc="upper left",
+ bbox_to_anchor=(0.08, 0.93),
+ ncol=1,
+ )
+ leg_line.get_frame().set_linewidth(0.0)
+ leg_line.get_frame().set_alpha(0.0)
+
+ fig.tight_layout()
+ fig.savefig(f"pism_kitp_gcm_{basin_name}_{res}.png", dpi=300)
+ fig.savefig(f"pism_kitp_gcm_{basin_name}_{res}.pdf")
+ plt.close(fig)
+
+ with mpl.rc_context(rc=rc_params):
+ for basin_name in baseline.basin.values:
+ basin_gcm = gcm_sub_baseline.sel(basin=basin_name)
+ basin_glf = (
+ gcm.sel(basin=basin_name)["grounding_line_flux"] * xr.DataArray(900**2).pint.quantify("m2")
+ ).pint.to("Gt/yr")
+ basin_slc = (basin_gcm["ice_mass"] * gt2mmsle).pint.dequantify()
+ time_vals = basin_slc.time.values
+
+ exps_palette = {k: v["color"] for k, v in EXPS_OPTS.items()}
+ gcm_palette = [v["color"] for v in EXPS_OPTS.values()]
+
+ fig, axs = plt.subplots(2, 1, figsize=(6.4, 4.8))
+
+ l = []
+
+ for exp_name, exp in EXPS_OPTS.items():
+ _gcm_slc = basin_slc.sel({"exp_id": exp_name})
+ _ = _gcm_slc.plot(ax=axs[0], hue="gcm_id", color=exp["color"], ls=exp["ls"], lw=0.75, add_legend=False)
+ _gcm_glf = basin_glf.sel({"exp_id": exp_name})
+ _ = _gcm_glf.plot(ax=axs[1], hue="gcm_id", color=exp["color"], ls=exp["ls"], lw=0.75, add_legend=False)
+ _l = _gcm_slc.isel({"gcm_id": 0}).plot(
+ ax=axs[0], color=exp["color"], ls=exp["ls"], label=exp["title"], lw=0.75, add_legend=False
+ )
+ l.append(_l)
+
+ axs[0].axhline(y=0, color="k", ls="dotted", lw=0.5)
+ axs[0].set_ylabel("Contribution to sea-level (mm)")
+ axs[1].set_xlabel("Time")
+ axs[0].set_title(basin_name)
+ axs[0].set_xlim(time_vals[0], time_vals[-1])
+ year_ticks = [t for t in time_vals if t.year % 50 == 0]
+ axs[1].set_xticks(year_ticks)
+ axs[1].set_xticklabels([f"{int(t.year)}" for t in year_ticks])
+
+ leg_line = fig.legend(
+ handles=[h for item in l for h in (item if isinstance(item, list) else [item])],
+ loc="upper left",
+ bbox_to_anchor=(0.08, 0.93),
+ ncol=1,
+ )
+ leg_line.get_frame().set_linewidth(0.0)
+ leg_line.get_frame().set_alpha(0.0)
+
+ fig.tight_layout()
+ fig.savefig(f"pism_kitp_norm_gcm_with_flux_{basin_name}_{res}.png", dpi=300)
+ fig.savefig(f"pism_kitp_norm_gcm_with_flux_{basin_name}_{res}.pdf")
+ plt.close(fig)
+
+ with mpl.rc_context(rc=rc_params):
+ for basin_name in baseline.basin.values:
+ basin_gcm = gcm_sub_baseline.sel(basin=basin_name)
+ basin_slc = (basin_gcm["ice_mass"] * gt2mmsle).pint.dequantify()
+ time_vals = basin_slc.time.values
+ basin_slice = (
+ basin_slc.sel({"time": slice(cftime.DatetimeNoLeap(90, 1, 1), cftime.DatetimeNoLeap(110, 1, 1))})
+ .mean(dim="time")
+ .squeeze()
+ )
+ basin_slice.name = "SLC"
+ _slice_df = basin_slice.to_dataframe()
+
+ exps_palette = {k: v["color"] for k, v in EXPS_OPTS.items()}
+ gcm_palette = [v["color"] for v in EXPS_OPTS.values()]
+
+ g = sns.catplot(
+ data=_slice_df,
+ kind="point",
+ y="exp_id",
+ x="SLC",
+ hue="gcm_id",
+ errorbar=None,
+ palette=gcm_palette,
+ alpha=0.6,
+ height=6,
+ )
+ g.despine(left=True)
+ g.set_axis_labels("Sea-level contribution (mm)", "")
+ g.legend.set_title("")
+ g.fig.savefig(f"pism_kitp_slice_yx_gcm_{basin_name}_{res}.png", dpi=300)
+ g.fig.savefig(f"pism_kitp_slice_yx_gcm_{basin_name}_{res}.pdf")
+
+ g = sns.catplot(
+ data=_slice_df,
+ kind="point",
+ y="gcm_id",
+ x="SLC",
+ hue="exp_id",
+ errorbar=None,
+ palette=exps_palette,
+ alpha=0.6,
+ height=6,
+ )
+ g.despine(left=True)
+ g.set_axis_labels("Sea-level contribution (mm)", "")
+ g.legend.set_title("")
+ g.fig.savefig(f"pism_kitp_slice_xy_gcm_{basin_name}_{res}.png", dpi=300)
+ g.fig.savefig(f"pism_kitp_slice_xy_gcm_{basin_name}_{res}.pdf")
+
+ fig, ax = plt.subplots(1, 1, figsize=(6.4, 3.6))
+
+ l = []
+
+ for exp_name, exp in EXPS_OPTS.items():
+ _gcm_slc = basin_slc.sel({"exp_id": exp_name})
+ _ = _gcm_slc.plot(ax=ax, hue="gcm_id", color=exp["color"], ls=exp["ls"], lw=0.75, add_legend=False)
+ _l = _gcm_slc.isel({"gcm_id": 0}).plot(
+ ax=ax, color=exp["color"], ls=exp["ls"], label=exp["title"], lw=0.75, add_legend=False
+ )
+ l.append(_l)
+
+ ax.axhline(y=0, color="k", ls="dotted", lw=0.5)
+ ax.set_ylabel("Contribution to sea-level (mm)")
+ ax.set_xlabel("Time")
+ ax.set_title(basin_name)
+ ax.set_xlim(time_vals[0], time_vals[-1])
+ year_ticks = [t for t in time_vals if t.year % 50 == 0]
+ ax.set_xticks(year_ticks)
+ ax.set_xticklabels([f"{int(t.year)}" for t in year_ticks])
+
+ leg_line = fig.legend(
+ handles=[h for item in l for h in (item if isinstance(item, list) else [item])],
+ loc="upper left",
+ bbox_to_anchor=(0.08, 0.93),
+ ncol=1,
+ )
+ leg_line.get_frame().set_linewidth(0.0)
+ leg_line.get_frame().set_alpha(0.0)
+
+ fig.tight_layout()
+ fig.savefig(f"pism_kitp_norm_gcm_{basin_name}_{res}.png", dpi=300)
+ fig.savefig(f"pism_kitp_norm_gcm_{basin_name}_{res}.pdf")
+ plt.close(fig)
+
+ # with mpl.rc_context(rc=rc_params):
+ # for basin_name in baseline.basin.values:
+ # basin_baseline = baseline_computed.sel(basin=basin_name)
+ # basin_gcm = gcm_computed.sel(basin=basin_name)
+
+ # fig, ax = plt.subplots(1, 1, figsize=(6.4, 3.6))
+
+ # l = []
+ # ci = []
+
+ # baseline_slc = (
+ # (basin_baseline["ice_mass"] - basin_baseline["ice_mass"].isel(time=0)) * gt2mmsle
+ # ).pint.dequantify()
+
+ # _l = baseline_slc.plot(
+ # ax=ax, color=BASELINE_OPTS["color"], ls=BASELINE_OPTS["ls"], label=BASELINE_OPTS["title"], lw=1
+ # )
+ # l.append(_l)
+
+ # gcm_slc = ((basin_gcm["ice_mass"] - basin_gcm["ice_mass"].isel(time=0)) * gt2mmsle).pint.dequantify()
+
+ # for _exp_id, _gcm_slc in gcm_slc.groupby("exp_id"):
+ # exp = EXPS_OPTS[_exp_id]
+ # _ = _gcm_slc.plot(ax=ax, hue="gcm_id", color=exp["color"], ls=exp["ls"], lw=0.75, add_legend=False)
+ # _l = _gcm_slc.isel({"gcm_id": 0, "exp_id": 0}).plot(
+ # ax=ax, color=exp["color"], ls=exp["ls"], label=exp["title"], lw=0.75, add_legend=False
+ # )
+ # l.append(_l)
+
+ # ax.axhline(y=0, color="k", ls="dotted", lw=0.5)
+ # ax.set_ylabel("Contribution to sea-level (mm)")
+ # ax.set_xlabel("Time")
+ # ax.set_title(basin_name)
+ # time_vals = baseline_slc.time.values
+ # ax.set_xlim(time_vals[0], time_vals[-1])
+ # year_ticks = [t for t in time_vals if t.year % 50 == 0]
+ # ax.set_xticks(year_ticks)
+ # ax.set_xticklabels([f"{int(t.year)}" for t in year_ticks])
+
+ # leg_line = fig.legend(
+ # handles=[h for item in l for h in (item if isinstance(item, list) else [item])],
+ # loc="upper left",
+ # bbox_to_anchor=(0.08, 0.93),
+ # ncol=1,
+ # )
+ # leg_line.get_frame().set_linewidth(0.0)
+ # leg_line.get_frame().set_alpha(0.0)
+
+ # fig.tight_layout()
+ # fig.savefig(f"pism_kitp_base_gcm_{basin_name}_{res}.png", dpi=300)
+ # fig.savefig(f"pism_kitp_base_gcm_{basin_name}_{res}.pdf")
+ # plt.close(fig)
+ # fig, axs = plt.subplots(2, 1, sharex=True, figsize=(6.4, 3.6), height_ratios=[1.618, 1])
+
+ # l = []
+ # ci = []
+
+ # ice_mass = basin_baseline["ice_mass"]
+ # ice_mass = ice_mass - ice_mass.isel(time=0)
+ # slc = ice_mass * gt2mmsle
+ # _l = slc.plot(
+ # ax=axs[0], color=BASELINE_OPTS["color"], ls=BASELINE_OPTS["ls"], label=BASELINE_OPTS["title"], lw=1
+ # )
+ # l.append(_l)
+
+ # smb = basin_baseline["tendency_of_ice_mass_due_to_surface_mass_flux"]
+ # smb.plot(ax=axs[1], color=BASELINE_OPTS["color"], ls=BASELINE_OPTS["ls"], lw=1)
+
+ # for exp_name, exp in EXPS_OPTS.items():
+ # in_multi = exp_name in basin_multi.exp_id.values
+ # in_single = exp_name in basin_single.exp_id.values
+ # if not in_multi and not in_single:
+ # continue
+
+ # if in_multi:
+ # multi_ds = basin_multi.sel(exp_id=exp_name)
+ # multi_ice_mass = multi_ds["ice_mass"]
+ # multi_ice_mass = multi_ice_mass - multi_ice_mass.isel(time=0)
+ # multi_slc = multi_ice_mass * gt2mmsle
+ # multi_smb = multi_ds["tendency_of_ice_mass_due_to_surface_mass_flux"]
+ # multi_glf = multi_ds["tendency_of_ice_mass_due_to_discharge"]
+ # multi_time_vals = multi_slc.sel(pctl=0.5).time.values
+
+ # if in_single:
+ # single_ds = basin_single.sel(exp_id=exp_name)
+ # single_ice_mass = single_ds["ice_mass"]
+ # single_ice_mass = single_ice_mass - single_ice_mass.isel(time=0)
+ # single_slc = single_ice_mass * gt2mmsle
+ # single_smb = single_ds["tendency_of_ice_mass_due_to_surface_mass_flux"]
+ # single_glf = single_ds["tendency_of_ice_mass_due_to_discharge"]
+ # single_time_vals = single_slc.sel(pctl=0.5).time.values
+
+ # if in_multi:
+ # _ci = axs[0].fill_between(
+ # multi_time_vals,
+ # multi_slc.sel(pctl=pctls[0]),
+ # multi_slc.sel(pctl=pctls[-1]),
+ # color=exp["color"],
+ # lw=0,
+ # alpha=0.25,
+ # )
+ # ci.append(_ci)
+
+ # _l = multi_slc.sel(pctl=0.5).plot(
+ # ax=axs[0], color=exp["color"], ls=exp["ls"], label=exp["title"], lw=0.75
+ # )
+ # l.append(_l)
+ # if in_single:
+ # _l = single_slc.sel(pctl=0.5).plot(
+ # ax=axs[0],
+ # color=exp["color"],
+ # ls=exp["ls"],
+ # label=exp["title"] if not in_multi else None,
+ # lw=0.75,
+ # )
+ # l.append(_l)
+ # single_smb.sel(pctl=0.5).plot(ax=axs[1], color=exp["color"], ls=exp["ls"], lw=0.75)
+ # if in_multi:
+ # axs[1].fill_between(
+ # multi_time_vals,
+ # multi_smb.sel(pctl=pctls[0]),
+ # multi_smb.sel(pctl=pctls[-1]),
+ # color=exp["color"],
+ # lw=0,
+ # alpha=0.25,
+ # )
+ # multi_smb.sel(pctl=0.5).plot(ax=axs[1], color=exp["color"], ls=exp["ls"], lw=0.75)
+
+ # axs[0].set_ylabel("Contribution to sea-level (mm)")
+ # axs[1].set_ylabel("Surface mass balance (Gt/yr)")
+ # axs[0].set_xlabel(None)
+ # axs[0].set_title(basin_name)
+ # axs[1].set_title(None)
+ # axs[1].axhline(y=0, color="k", ls="dotted", lw=0.5)
+ # axs[-1].set_xlim(multi_time_vals[0], multi_time_vals[-1])
+ # # cftime axis: pick every 50th year from the actual time values
+ # year_ticks = [t for t in multi_time_vals if t.year % 50 == 0]
+ # axs[-1].set_xticks(year_ticks)
+ # axs[-1].set_xticklabels([f"{int(t.year)}" for t in year_ticks])
+ # # Legend 1: multi-GCM confidence intervals
+
+ # # leg_ci = fig.legend(
+ # # handles=ci,
+ # # loc="upper left",
+ # # bbox_to_anchor=(0.08, 0.93),
+ # # ncol=1,
+ # # title="5-95% c.i.",
+ # # )
+ # # leg_ci.get_frame().set_linewidth(0.0)
+ # # leg_ci.get_frame().set_alpha(0.0)
+ # # fig.add_artist(leg_ci)
+
+ # leg_line = fig.legend(
+ # handles=[h for item in l for h in (item if isinstance(item, list) else [item])],
+ # loc="upper left",
+ # bbox_to_anchor=(0.08, 0.93),
+ # ncol=1,
+ # )
+ # leg_line.get_frame().set_linewidth(0.0)
+ # leg_line.get_frame().set_alpha(0.0)
+
+ # fig.tight_layout()
+ # fig.savefig(f"pism_kitp_multi_gcm_{basin_name}_{res}.png", dpi=300)
+ # fig.savefig(f"pism_kitp_multi_gcm_{basin_name}_{res}.pdf")
+ # plt.close(fig)
+
+ # fig, axs = plt.subplots(2, 1, sharex=True, figsize=(6.4, 3.6), height_ratios=[1.618, 1])
+
+ # ice_mass_bl = basin_baseline["ice_mass"]
+ # ice_mass_bl = ice_mass_bl - ice_mass_bl.isel(time=0)
+ # slc_bl = ice_mass_bl * gt2mmsle
+ # slc_bl.plot(
+ # ax=axs[0], color=BASELINE_OPTS["color"], ls=BASELINE_OPTS["ls"], label=BASELINE_OPTS["title"], lw=1
+ # )
+
+ # smb_bl = basin_baseline["tendency_of_ice_mass_due_to_surface_mass_flux"]
+ # smb_bl.plot(ax=axs[1], color=BASELINE_OPTS["color"], ls=BASELINE_OPTS["ls"], lw=1)
+
+ # ice_mass_gcm = basin_single_gcm["ice_mass"].pint.to("Gt").pint.dequantify()
+ # ice_mass_gcm = ice_mass_gcm - ice_mass_gcm.isel(time=0)
+ # slc_gcm = ice_mass_gcm * gt2mmsle.pint.dequantify()
+ # smb_gcm = (
+ # basin_single_gcm["tendency_of_ice_mass_due_to_surface_mass_flux"].pint.to("Gt/yr").pint.dequantify()
+ # )
+
+ # for exp_name, exp in EXPS_OPTS.items():
+
+ # slc_gcm.sel({"exp_id": exp_name}).plot(
+ # ax=axs[0], color=exp["color"], ls=exp["ls"], label=exp["title"], lw=0.75
+ # )
+ # smb_gcm.sel({"exp_id": exp_name}).plot(
+ # ax=axs[1], color=exp["color"], ls=exp["ls"], add_legend=False, lw=0.75
+ # )
+
+ # handles, labels = axs[0].get_legend_handles_labels()
+ # legend_main = fig.legend(handles, labels, loc="upper left", bbox_to_anchor=(0.1, 0.9), ncol=1)
+ # legend_main.set_title(None)
+ # legend_main.get_frame().set_linewidth(0.0)
+ # legend_main.get_frame().set_alpha(0.0)
+ # axs[0].set_ylabel("Contribution to sea-level (mm)")
+ # axs[1].set_ylabel("Surface mass balance (Gt/yr)")
+ # axs[0].set_xlabel(None)
+ # axs[0].set_title(basin_name)
+ # axs[1].set_title(None)
+ # axs[1].axhline(y=0, color="k", ls="dotted", lw=0.5)
+ # axs[-1].set_xlim(multi_time_vals[0], multi_time_vals[-1])
+ # # cftime axis: pick every 50th year from the actual time values
+ # year_ticks = [t for t in multi_time_vals if t.year % 50 == 0]
+ # axs[-1].set_xticks(year_ticks)
+ # axs[-1].set_xticklabels([f"{int(t.year)}" for t in year_ticks])
+ # # Legend 1: multi-GCM confidence intervals
+ # fig.tight_layout()
+ # fig.savefig(f"pism_kitp_cesm1_gcm_{basin_name}_{res}.png", dpi=300)
+ # fig.savefig(f"pism_kitp_cesm1_gcm_{basin_name}_{res}.pdf")
+ # plt.close(fig)
+
+ # fig, axs = plt.subplots(4, 2, sharex=True, sharey=False, figsize=(6.4, 4.8))
+
+ # for k, basin_name in enumerate(baseline.basin):
+ # ax = axs.flatten()[k]
+ # ice_mass_bl = baseline.sel(basin=basin_name)["ice_mass"].pint.to("Gt").pint.dequantify()
+ # ice_mass_bl -= ice_mass_bl.isel(time=0)
+ # ice_mass_bl.plot(
+ # ax=ax, color=BASELINE_OPTS["color"], ls=BASELINE_OPTS["ls"], label=BASELINE_OPTS["title"], lw=1
+ # )
+
+ # for exp_name, exp in EXPS_OPTS.items():
+ # ice_mass_exp = (
+ # single_gcm.sel(basin=basin_name).sel(exp_id=exp_name)["ice_mass"].pint.to("Gt").pint.dequantify()
+ # )
+ # ice_mass_exp -= ice_mass_exp.isel(time=0)
+ # ice_mass_exp.plot(ax=ax, color=exp["color"], ls=exp["ls"], label=exp["title"], lw=0.75)
+ # ax.set_title(basin_name.values)
+ # ax.axhline(y=0, color="k", ls="dotted", lw=0.5)
+ # ax.set_xlabel("")
+ # ax.set_ylabel("")
+ # handles, labels = axs[0, 0].get_legend_handles_labels()
+ # legend_main = fig.legend(handles, labels, loc="upper left", bbox_to_anchor=(0.1, 0.9), ncol=1)
+ # legend_main.set_title(None)
+ # legend_main.get_frame().set_linewidth(0.0)
+ # legend_main.get_frame().set_alpha(0.0)
+
+ # fig.supxlabel("Time")
+ # fig.supylabel("Ice mass change (Gt)")
+ # fig.savefig(f"pism_kitp_{res}.png", dpi=300)
+ # fig.savefig(f"pism_kitp_{res}.pdf")
+ # plt.close(fig)
+
+
+def main():
+ """Run main script."""
+ parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+ parser.description = "Postprocess KITP Greenland."
+ parser.add_argument("--ntasks", help="Sets number of tasks.", type=int, default=4)
+ parser.add_argument("FILES", help="netCDF files to process.", nargs="*")
+
+ options, _ = parser.parse_known_args()
+ infiles = options.FILES
+
+ log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+ logging.basicConfig(level=logging.WARNING, format=log_format)
+
+ pism_logger = logging.getLogger("pism_terra")
+ pism_logger.setLevel(logging.INFO)
+ pism_logger.propagate = False
+
+ console_handler = logging.StreamHandler()
+ console_handler.setLevel(logging.INFO)
+ console_handler.setFormatter(logging.Formatter(log_format))
+ pism_logger.addHandler(console_handler)
+
+ file_handler = logging.FileHandler("analyze.log")
+ file_handler.setLevel(logging.INFO)
+ file_handler.setFormatter(logging.Formatter(log_format))
+ pism_logger.addHandler(file_handler)
+
+ plot_scalar_timeseries(infiles)
+
+
+if __name__ == "__main__":
+ __spec__ = None # type: ignore
+ main()
diff --git a/pism_terra/kitp/calibrate.py b/pism_terra/kitp/calibrate.py
new file mode 100644
index 0000000..ce1cee3
--- /dev/null
+++ b/pism_terra/kitp/calibrate.py
@@ -0,0 +1,362 @@
+"""
+KITP calibration driver.
+
+Ranks PISM-KITP UQ ensemble members against observed surface mass-balance
+fields with a spatially-aware metric: pixel-wise RMSE is replaced by a
+block-bootstrap RMSE whose block size matches the field's decorrelation
+length. The best-RMSE experiment and every experiment whose 5-95 % CI
+overlaps the leader's are reported as the "tied" calibration set.
+"""
+
+import json
+from functools import partial
+
+import matplotlib.pylab as plt
+import numpy as np
+import pandas as pd
+import pint_xarray # pylint: disable=unused-import
+import xarray as xr
+import xarray_regrid.methods.conservative # pylint: disable=unused-import
+from dask.diagnostics import ProgressBar
+
+from pism_terra.filtering import importance_sampling
+from pism_terra.processing import preprocess_netcdf as preprocess
+
+debm_uq_vars = {
+ "surface.debm_simple.c1": "c1",
+ "surface.debm_simple.c2": "c2",
+ "surface.debm_simple.air_temp_all_precip_as_snow": "as_snow",
+ "surface.debm_simple.air_temp_all_precip_as_rain": "as_rain",
+ "surface.debm_simple.refreeze": "refreeze",
+}
+
+
+def decorrelation_length(field_2d, pixel_size, threshold=1.0 / np.e):
+ """
+ Radially-averaged spatial-ACF decorrelation length for a 2D field.
+
+ Pixel-wise RMSE treats every cell as independent, but glaciological
+ fields are smooth on scales of many cells. The lag at which the
+ radially-averaged autocorrelation first falls below ``threshold`` is a
+ practical block side for bootstrap resampling: blocks of that side are
+ statistically (approximately) independent.
+
+ Parameters
+ ----------
+ field_2d : numpy.ndarray
+ The two-dimensional field to analyse. Non-finite values are filled
+ with the field's mean before the FFT; if every entry is non-finite
+ the function returns ``nan``.
+ pixel_size : float
+ Side length of one cell in physical units (typically metres). The
+ returned decorrelation length is in the same units.
+ threshold : float, default ``1 / e``
+ ACF level at which the decorrelation length is read off. Common
+ alternatives are ``0.1`` (longer block) or ``0.5`` (shorter block).
+
+ Returns
+ -------
+ float
+ Decorrelation length in the units of ``pixel_size``. Returns
+ ``nan`` when the input has no finite values.
+ """
+ a = np.asarray(field_2d, dtype=float)
+ finite = np.isfinite(a)
+ if not finite.any():
+ return float("nan")
+ a = np.where(finite, a, np.nanmean(a))
+ a = a - a.mean()
+ fft = np.fft.fft2(a)
+ acf = np.fft.fftshift(np.fft.ifft2(fft * np.conj(fft)).real)
+ acf = acf / acf.max()
+ ny, nx = a.shape
+ cy, cx = ny // 2, nx // 2
+ yy, xx = np.indices(a.shape)
+ r = np.sqrt((yy - cy) ** 2 + (xx - cx) ** 2).astype(int)
+ counts = np.maximum(np.bincount(r.ravel()), 1)
+ radial = np.bincount(r.ravel(), weights=acf.ravel()) / counts
+ rmax = min(cy, cx)
+ radial = radial[: rmax + 1]
+ below = np.where(radial < threshold)[0]
+ lag_pixels = below[0] if below.size else rmax
+ return float(lag_pixels) * float(pixel_size)
+
+
+def block_bootstrap_rmse(sim, obs, block_size, n_boot=500, seed=0):
+ """
+ Block-bootstrap spatial RMSE per experiment.
+
+ The domain is tiled into non-overlapping square blocks of side
+ ``block_size`` pixels. For each bootstrap iteration, blocks are drawn
+ with replacement and a single global RMSE is computed across the
+ resampled blocks for every experiment in ``sim``. Choosing
+ ``block_size`` ≳ ``decorrelation_length(obs) / pixel_size`` makes the
+ resampled blocks (approximately) independent, so the spread of
+ bootstrap RMSEs reflects sampling uncertainty under spatial
+ autocorrelation.
+
+ Parameters
+ ----------
+ sim : xarray.DataArray
+ Per-experiment simulated field with dims ``(exp_id, y, x)``.
+ obs : xarray.DataArray
+ Observed field with dims ``(y, x)`` aligned with ``sim``.
+ block_size : int
+ Block side in pixels. Must be ≥ 1; typically chosen as
+ ``ceil(L / pixel_size)`` where ``L`` is the decorrelation length.
+ n_boot : int, default ``500``
+ Number of bootstrap resamples.
+ seed : int, default ``0``
+ Seed for :class:`numpy.random.Generator`. Use a fixed value to
+ make the bootstrap deterministic.
+
+ Returns
+ -------
+ xarray.DataArray
+ RMSE distribution with dims ``(exp_id, boot)``, where ``boot``
+ ranges over the bootstrap resamples. Aggregate with
+ ``.mean(dim="boot")`` for the central RMSE and
+ ``.quantile([0.05, 0.95], dim="boot")`` for confidence bands.
+ """
+ sim_v = np.asarray(sim.values, dtype=float)
+ obs_v = np.asarray(obs.values, dtype=float)
+ sq_err = (sim_v - obs_v[None, :, :]) ** 2
+ valid = np.isfinite(sq_err).all(axis=0)
+ ny, nx = obs_v.shape
+ by = max(1, ny // block_size)
+ bx = max(1, nx // block_size)
+ block_sums = np.zeros((sim_v.shape[0], by, bx))
+ block_counts = np.zeros((by, bx), dtype=int)
+ for i in range(by):
+ for j in range(bx):
+ ys = slice(i * block_size, (i + 1) * block_size)
+ xs = slice(j * block_size, (j + 1) * block_size)
+ v = valid[ys, xs]
+ block_counts[i, j] = int(v.sum())
+ chunk = np.where(v, sq_err[:, ys, xs], 0.0)
+ block_sums[:, i, j] = chunk.sum(axis=(1, 2))
+ block_sums = block_sums.reshape(sim_v.shape[0], -1)
+ block_counts = block_counts.reshape(-1)
+ valid_blocks = np.where(block_counts > 0)[0]
+ rng = np.random.default_rng(seed)
+ rmses = np.empty((sim_v.shape[0], n_boot))
+ for b in range(n_boot):
+ idx = rng.choice(valid_blocks, size=valid_blocks.size, replace=True)
+ s = block_sums[:, idx].sum(axis=1)
+ c = block_counts[idx].sum()
+ rmses[:, b] = np.sqrt(s / max(c, 1))
+ return xr.DataArray(
+ rmses,
+ dims=["exp_id", "boot"],
+ coords={"exp_id": sim.exp_id, "boot": np.arange(n_boot)},
+ )
+
+
+data_dir = "~/base/pism-terra"
+
+pctls = [0.05, 0.95]
+fontsize = 6
+rc_params = {
+ "axes.linewidth": 0.15,
+ "xtick.major.size": 2.0,
+ "xtick.major.width": 0.15,
+ "ytick.major.size": 2.0,
+ "ytick.major.width": 0.15,
+ "hatch.linewidth": 0.15,
+ "font.size": fontsize,
+ "font.family": "DejaVu Sans",
+}
+
+debm_uq_vars = {
+ "surface.debm_simple.c1": "c1",
+ "surface.debm_simple.c2": "c2",
+ "surface.debm_simple.air_temp_all_precip_as_snow": "as_snow",
+ "surface.debm_simple.air_temp_all_precip_as_rain": "as_rain",
+ "surface.debm_simple.refreeze": "refreeze",
+}
+
+pdd_uq_vars = {"surface.pdd.factor_ice": "fice", "surface.pdd.factor_snow": "fsnow", "surface.pdd.refreeze": "refreeze"}
+
+m_vars = ["surface_melt_flux", "surface_runoff_flux", "climatic_mass_balance"]
+
+obs = xr.open_dataset(
+ f"{data_dir}/2026_06_kitp_debm_calib/kitp/input/v4/HIRHAM5-ERA5_YMM_1990_2019_v4.nc",
+ engine="netcdf4",
+ decode_times=False,
+ decode_timedelta=False,
+ chunks=None,
+).drop_dims("nv", errors="ignore")
+
+obs = obs.pint.quantify()
+for v in ["surface_melt_flux", "surface_runoff_flux", "climatic_mass_balance"]:
+ obs[v] = obs[v].pint.to("kg m^-2 yr^-1")
+obs = obs.pint.dequantify()
+for v in ["surface_melt_flux", "surface_runoff_flux", "climatic_mass_balance"]:
+ obs[f"{v}_error"] = xr.where(obs[v] != 0, 0.10 * obs[v], 1e-8)
+
+for (
+ ebm,
+ ebm_uq_vars,
+) in zip(["debm"], [debm_uq_vars]):
+
+ ds = (
+ xr.open_mfdataset(
+ f"{data_dir}/2026_06_kitp_{ebm}_calib/output/processed_spatial/clipped_spatial_g4800m_id_HIRHAM5-ERA5_YMM_1990_2019_uq_*.nc",
+ preprocess=partial(preprocess, uq_regexp=None, exp_regexp="uq_(.+?)_"),
+ engine="netcdf4",
+ join="outer",
+ compat="no_conflicts",
+ parallel=True,
+ chunks="auto",
+ decode_times=False,
+ decode_timedelta=False,
+ )
+ .drop_dims("nv", errors="ignore")
+ .pint.quantify()
+ )
+ ds["exp_id"] = ds["exp_id"].astype("int")
+ for v in ["surface_melt_flux", "surface_runoff_flux", "climatic_mass_balance"]:
+ ds[v] = ds[v].pint.to("kg m^-2 yr^-1")
+ ds = ds.pint.dequantify()
+
+ ebm_uq_df = ds.pism_config.to_series().apply(json.loads).apply(pd.Series)[ebm_uq_vars.keys()]
+ ds["time"] = obs["time"]
+
+ _obs = obs.regrid.conservative(ds.drop_vars("pism_config")).squeeze()
+ mask = ds[m_vars].isel(exp_id=0).notnull()
+ _obs[m_vars] = _obs[m_vars].where(mask)
+ melt_mask = _obs["climatic_mass_balance"].mean(dim="time") < 1e36
+ _obs[m_vars] = _obs[m_vars].where(melt_mask)
+ _ds = ds[m_vars].where(melt_mask)
+
+ for v in ["climatic_mass_balance"]:
+
+ for fudge_factor in [1, 10, 100, 1000, 10_000, 100_000]:
+ with ProgressBar():
+ ebm_filtered = importance_sampling(
+ _ds,
+ _obs,
+ sim_var=v,
+ obs_mean_var=v,
+ obs_std_var=f"{v}_error",
+ sum_dims=["time", "x", "y"],
+ n_samples=ds.exp_id.size,
+ fudge_factor=fudge_factor,
+ )
+
+ ebm_sampled_ids = ebm_filtered.exp_id_sampled.values
+ ebm_counts = pd.Series(ebm_sampled_ids).value_counts()
+
+ # Reindex config_df to the sampled IDs and plot histograms
+ ds_sampled_configs = ebm_uq_df.loc[ebm_counts.index].reindex(ebm_counts.index)
+ most_sampled_id = ebm_counts.idxmax()
+ most_sampled_params = ebm_uq_df.loc[most_sampled_id]
+ print(f"\n{ebm} / {v} — {fudge_factor} — most sampled id={most_sampled_id} (count={ebm_counts.max()})")
+ for k, short in ebm_uq_vars.items():
+ print(f" {short}: {most_sampled_params[k]:.6g}")
+
+ fig, axes = plt.subplots(1, len(ebm_uq_vars), sharey=True, figsize=(6.4, 1.8))
+ for ax, (key, value) in zip(axes.flat, ebm_uq_vars.items()):
+ # Repeat each parameter value by its sample count
+ values = np.repeat(
+ ebm_uq_df[key].values, ebm_counts.reindex(ebm_uq_df.index, fill_value=0).values.astype(int)
+ )
+ print(key, np.median(values))
+ ax.hist(values, bins=15)
+ ax.set_xlabel(value)
+ ax.set_xlim(ebm_uq_df[key].min(), ebm_uq_df[key].max())
+ # ax.set_ylabel("Count")
+
+ fig.tight_layout()
+ fig.savefig(f"{ebm}_{v}_ff_{fudge_factor}.png", dpi=300)
+ plt.close()
+ del fig
+
+ with ProgressBar():
+
+ # 1) Decorrelation length from the observed time-mean field.
+ sim_mean_all = _ds[v].mean(dim="time").compute()
+ obs_mean = _obs[v].mean(dim="time").squeeze().compute()
+ pixel_size = float(abs(_obs.x.diff("x").mean()))
+ L = decorrelation_length(obs_mean.values, pixel_size)
+ block_size = max(1, int(np.ceil(L / pixel_size)))
+ print(f"{ebm}/{v}: decorrelation length ≈ {L:.0f} m, block_size = {block_size} px")
+
+ # 2) Block-bootstrap RMSE per exp_id (honors spatial correlation).
+ rmse_boot = block_bootstrap_rmse(sim_mean_all, obs_mean, block_size, n_boot=500)
+ rmse_mean = rmse_boot.mean(dim="boot")
+ rmse_lo = rmse_boot.quantile(0.05, dim="boot")
+ rmse_hi = rmse_boot.quantile(0.95, dim="boot")
+
+ # 3) Rank by bootstrap-mean RMSE; treat exp_ids whose CI overlaps
+ # the leader's upper bound as statistically tied with the best.
+ best_id = rmse_mean.idxmin(dim="exp_id").values
+ best_hi = float(rmse_hi.sel(exp_id=best_id))
+ tied_mask = rmse_lo <= best_hi
+ tied_ids = list(rmse_mean.exp_id.where(tied_mask, drop=True).values)
+ print(f"{ebm}/{v}: best exp_id = {best_id}, n tied within 5-95% CI = {len(tied_ids)}")
+
+ # Per-experiment weight for the parameter histograms: 1 if the
+ # exp_id is in the statistically-tied set, 0 otherwise. This is
+ # what ``np.repeat`` consumes below so each parameter value
+ # contributes to the histogram only if its experiment passed the
+ # bootstrap tie test.
+ ebm_counts = pd.Series(
+ tied_mask.values.astype(int),
+ index=pd.Index(rmse_mean.exp_id.values, name="exp_id"),
+ )
+
+ fig, axes = plt.subplots(1, len(ebm_uq_vars), sharey=True, figsize=(6.4, 1.8))
+ for ax, (key, value) in zip(axes.flat, ebm_uq_vars.items()):
+ # Repeat each parameter value by its sample count (= 1 if
+ # the experiment tied with the best, 0 otherwise).
+ values = np.repeat(
+ ebm_uq_df[key].values, ebm_counts.reindex(ebm_uq_df.index, fill_value=0).values.astype(int)
+ )
+ ax.hist(values, bins=15)
+ ax.set_xlabel(value)
+ ax.set_xlim(ebm_uq_df[key].min(), ebm_uq_df[key].max())
+ # ax.set_ylabel("Count")
+ fig.tight_layout()
+ fig.savefig(f"{ebm}_{v}.png", dpi=300)
+ plt.close()
+ del fig
+
+ # Write per-experiment stats to CSV so the user can inspect ties.
+ rmse_df = (
+ pd.DataFrame(
+ {
+ "rmse_mean": rmse_mean.values,
+ "rmse_lo": rmse_lo.values,
+ "rmse_hi": rmse_hi.values,
+ "tied_with_best": tied_mask.values,
+ },
+ index=pd.Index(rmse_mean.exp_id.values, name="exp_id"),
+ )
+ .join(ebm_uq_df, how="left")
+ .sort_values("rmse_mean")
+ )
+ rmse_df.to_csv(f"{ebm}_{v}_rmse.csv")
+
+ sim_best = _ds[v].sel(exp_id=best_id).mean(dim="time").squeeze()
+ vmin = min(float(obs_mean.min()), float(sim_best.min()))
+ vmax = max(float(obs_mean.max()), float(sim_best.max()))
+ best_params = ebm_uq_df.loc[best_id]
+ fig, axes = plt.subplots(1, 3, sharey=True, figsize=(12, 4))
+ obs_mean.plot(ax=axes[0], vmin=vmin, vmax=vmax)
+ axes[0].set_title("Observed")
+ sim_best.plot(ax=axes[1], vmin=vmin, vmax=vmax)
+ param_str = ", ".join(f"{v}={best_params[k]:.4g}" for k, v in ebm_uq_vars.items())
+ rmse_best_mean = float(rmse_mean.sel(exp_id=best_id))
+ rmse_best_lo = float(rmse_lo.sel(exp_id=best_id))
+ rmse_best_hi = float(rmse_hi.sel(exp_id=best_id))
+ axes[1].set_title(
+ f"Best (id={best_id}, RMSE={rmse_best_mean:.1f} "
+ f"[{rmse_best_lo:.1f}-{rmse_best_hi:.1f}], n_tied={len(tied_ids)})\n{param_str}"
+ )
+ (sim_best - obs_mean).plot(ax=axes[2], cmap="RdBu", vmin=-1000, vmax=1000)
+ axes[2].set_title("Difference")
+ fig.tight_layout()
+ fig.savefig(f"{ebm}_{v}_best_rmse.png", dpi=300)
+ plt.close()
+ del fig
diff --git a/pism_terra/kitp/forcing.py b/pism_terra/kitp/forcing.py
index a56cac8..37b1f51 100644
--- a/pism_terra/kitp/forcing.py
+++ b/pism_terra/kitp/forcing.py
@@ -22,12 +22,14 @@
Prepare KITP Greenland data sets.
"""
+import logging
import os
import re
+import shutil
import tempfile
import time
from argparse import ArgumentParser
-from collections.abc import Sequence
+from collections.abc import Iterable, Sequence
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import as_completed as cf_as_completed
from pathlib import Path
@@ -38,6 +40,7 @@
import geopandas as gpd
import numpy as np
import pandas as pd
+import pint_xarray
import rioxarray # pylint: disable=unused-import
import toml
import xarray as xr
@@ -45,78 +48,330 @@
from cdo import Cdo
from dask.distributed import Client, as_completed
from pyfiglet import Figlet
+from pyproj import Proj
+from rasterio.enums import Resampling
from tqdm import tqdm
from pism_terra.aws import s3_to_local
from pism_terra.domain import create_domain
-from pism_terra.download import download_hirham, unzip_files
-from pism_terra.raster import create_ds
+from pism_terra.download import carra_download_request, download_hirham, unzip_files
+from pism_terra.grids import load_grid
+from pism_terra.raster import add_time_bounds, create_ds
from pism_terra.vector import dissolve
from pism_terra.workflow import check_xr_fully, check_xr_lazy
+logger = logging.getLogger(__name__)
+
xr.set_options(keep_attrs=True)
-hirham_grid = """
-gridtype = projection
-xname = rlon
-xsize = 402
-ysize = 602
-xfirst = -16
-xinc = 0.05
-xunits = degree
-yname = rlat
-yfirst = -14
-yinc = 0.05
-yunits = degree
-grid_mapping_name = rotated_latitude_longitude
-grid_north_pole_longitude = 160
-grid_north_pole_latitude = 18
-north_pole_grid_longitude = 0
-"""
+hirham_grid = load_grid("hirham")
+bedmachine_grid = load_grid("bedmachine")
+carra2_grid = load_grid("carra2")
+ismip6_grid = load_grid("ismip6")
-bedmachine_grid = """
-gridtype = projection
-xsize = 10218
-ysize = 18346
-xunits = "meter"
-yunits = "meter"
-xfirst = -65300
-xinc = 150
-yfirst = -3384425
-yinc = 150
-grid_mapping = crs
-grid_mapping_name = polar_stereographic
-straight_vertical_longitude_from_pole = -39.
-standard_parallel = 71.
-latitude_of_projection_origin = 90.
-false_easting = 0.
-false_northing = 0.
-"""
-ismip6_grid = """
-gridtype = projection
-xsize = 1496
-ysize = 2700
-xunits = "meter"
-yunits = "meter"
-xfirst = -640000
-xinc = 1000
-yfirst = -3355000
-yinc = 1000
-grid_mapping = crs
-grid_mapping_name = polar_stereographic
-straight_vertical_longitude_from_pole = -45.
-standard_parallel = 70.
-latitude_of_projection_origin = 90.
-false_easting = 0.
-false_northing = 0.
-"""
+def process_carra2(
+ data_dir: str | Path,
+ output_file: str | Path,
+ year: list[str | int] = [
+ "1986",
+ "1987",
+ "1988",
+ "1991",
+ "1992",
+ "1993",
+ "1996",
+ "1997",
+ "1998",
+ "2001",
+ "2002",
+ "2003",
+ "2006",
+ "2007",
+ "2008",
+ "2011",
+ "2012",
+ "2013",
+ "2016",
+ "2017",
+ "2018",
+ "2021",
+ "2022",
+ "2023",
+ ],
+ max_workers: int = 8,
+ force_overwrite: bool = False,
+ **kwargs,
+):
+ """
+ Download monthly CARRA2 reanalysis and write a NetCDF.
+ Parameters
+ ----------
+ data_dir : Union[str, Path]
+ Directory containing the input data.
+ output_file : Union[str, Path]
+ Path to the output NetCDF file.
+ year : list[str | int]
+ List of years to download.
+ max_workers : int, default 8
+ Maximum number of concurrent CDS download requests.
+ force_overwrite : bool, default False
+ If ``True``, recompute intermediate and output files even if they exist.
+ **kwargs
+ Additional keyword arguments forwarded to :func:`download_request`
+ (e.g., alternate ``variable`` sequences, custom authentication/session
+ options, or client settings). These are passed unchanged to the CDS
+ retrieval helper.
+
+ Notes
+ -----
+ - Output variables:
+ - ``air_temp`` (K) from CARRA ``t2m``.
+ - ``precipitation`` (kg m^-2 day^-1) from CARRA ``tp`` (converted).
+ - ``time_bounds`` are added for CF-style climatological metadata.
+ - If missing values are detected in the regional subset, the function
+ patches them from the global reanalysis (same period).
+ """
-def process_hirham_cdo(
- data_dir: str | Path,
+ print("")
+ print("Generate historical climate")
+ print("-" * 120)
+
+ carra2_path = data_dir / Path("carra2")
+ carra2_path.mkdir(exist_ok=True)
+ carra2_grid_path = carra2_path / Path("carra2_grid.txt")
+ carra2_grid_path.write_text(carra2_grid)
+ target_grid_path = carra2_path / Path("ismip6_grid.txt")
+ target_grid_path.write_text(ismip6_grid)
+
+ precipitation_dataset = "reanalysis-pan-carra-means"
+ precipitation_request = {
+ "time_aggregation": "monthly",
+ "level_type": "single_levels",
+ "variable": ["total_precipitation"],
+ "product_type": "forecast_based",
+ "year": year,
+ "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"],
+ "data_format": "netcdf",
+ "area": [90, -180, 40, 180],
+ }
+
+ precipitation_files = carra_download_request(
+ precipitation_dataset,
+ precipitation_request,
+ file_path=carra2_path / Path("pr.nc"),
+ max_workers=max_workers,
+ **kwargs, # pass the full CARRA request dict
+ )
+
+ temperature_dataset = "reanalysis-pan-carra-means"
+ temperature_request = {
+ "time_aggregation": "daily",
+ "level_type": "single_levels",
+ "variable": ["2m_temperature"],
+ "product_type": "analysis_based",
+ "year": year,
+ "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"],
+ "day": [
+ "01",
+ "02",
+ "03",
+ "04",
+ "05",
+ "06",
+ "07",
+ "08",
+ "09",
+ "10",
+ "11",
+ "12",
+ "13",
+ "14",
+ "15",
+ "16",
+ "17",
+ "18",
+ "19",
+ "20",
+ "21",
+ "22",
+ "23",
+ "24",
+ "25",
+ "26",
+ "27",
+ "28",
+ "29",
+ "30",
+ "31",
+ ],
+ "data_format": "netcdf",
+ "area": [90, -180, 40, 180],
+ }
+
+ temperature_files = carra_download_request(
+ temperature_dataset,
+ temperature_request,
+ file_path=carra2_path / Path("tas.nc"),
+ max_workers=max_workers,
+ **kwargs, # pass the full CARRA request dict
+ )
+
+ logger.info(
+ "Downloaded %d precipitation files, %d temperature files", len(precipitation_files), len(temperature_files)
+ )
+
+ grid = str(carra2_grid_path.resolve())
+ cdo = Cdo()
+ cdo.debug = True
+
+ # --- Step 1: per-year batches (setgrid, settaxis, monmean/monstd) ---
+
+ pr_sorted = sorted(precipitation_files)
+ tas_sorted = sorted(temperature_files)
+
+ batches = []
+ for yr, pr_f, tas_f in zip(year, pr_sorted, tas_sorted):
+ batch_out = str((carra2_path / f"batch_{yr}.nc").resolve())
+ batches.append((yr, str(pr_f), str(tas_f), batch_out))
+
+ def _process_carra2_batch(args):
+ """
+ Process a single year-batch: fix grid/time, compute monmean/monstd, merge.
+
+ Parameters
+ ----------
+ args : tuple
+ A ``(yr, pr_f, tas_f, batch_out)`` tuple with the year string,
+ precipitation file, temperature file, and output path.
+
+ Returns
+ -------
+ str
+ Path to the merged output file.
+ """
+ yr, pr_f, tas_f, batch_out = args
+ tmp = carra2_path
+ cdo_local = Cdo(tempdir=tmp)
+
+ # Precipitation: monthly means (already monthly data, just fix grid + time)
+ pr_fixed = os.path.join(tmp, f"pr_{yr}.nc")
+ cdo_local.setgrid(
+ grid,
+ input=f"""-setattribute,precipitation@units="kg m^-2 day^-1" -chname,tp,precipitation """
+ f"""-settbounds,1mon -setreftime,{yr}-01-01 -settunits,days -settaxis,{yr}-01-15,00:00:00,1mon {pr_f}""",
+ output=pr_fixed,
+ options="--reduce_dim -f nc4 -z zip_2",
+ )
+
+ # Temperature: aggregate daily -> monthly mean
+ tas_mm = os.path.join(tmp, f"tas_mm_{yr}.nc")
+ cdo_local.monmean(
+ input=f"""-setgrid,{grid} -chname,t2m,air_temp """
+ f"""-settbounds,1day -setreftime,{yr}-01-01 -settunits,days -settaxis,{yr}-01-01,00:00:00,1day {tas_f}""",
+ output=tas_mm,
+ options="--reduce_dim -f nc4 -z zip_2",
+ )
+
+ # Temperature: aggregate daily -> monthly std
+ tas_mstd = os.path.join(tmp, f"tas_mstd_{yr}.nc")
+ cdo_local.setattribute(
+ """air_temp_sd@long_name="standard deviation of 2-m air temperature" """,
+ input=f"""-chname,air_temp,air_temp_sd -monstd -setgrid,{grid} -chname,t2m,air_temp """
+ f"""-settbounds,1day -setreftime,{yr}-01-01 -settunits,days -settaxis,{yr}-01-01,00:00:00,1day {tas_f}""",
+ output=tas_mstd,
+ options="--reduce_dim -f nc4 -z zip_2",
+ )
+
+ # Merge pr + tas_mm + tas_mstd for this year
+ cdo_local.merge(
+ input=f"{pr_fixed} {tas_mm} {tas_mstd}",
+ output=batch_out,
+ options="-f nc4 -z zip_2",
+ )
+ # Clean up per-year intermediate files only (not the shared directory)
+ for f in (pr_fixed, tas_mm, tas_mstd):
+ Path(f).unlink(missing_ok=True)
+ return batch_out
+
+ # Only process batches that don't already exist (unless force_overwrite)
+ batches_to_run = [b for b in batches if (not check_xr_lazy(b[3])) or force_overwrite]
+ if batches_to_run:
+ logger.info("CDO: processing %d year batches (setgrid + monmean/monstd)...", len(batches_to_run))
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ futures = {executor.submit(_process_carra2_batch, b): b for b in batches_to_run}
+ for future in tqdm(cf_as_completed(futures), total=len(futures), desc="Processing CARRA2 batches"):
+ future.result()
+ else:
+ logger.info("CDO: all %d year batches already exist, skipping.", len(batches))
+
+ batch_files = sorted(b[3] for b in batches)
+
+ # --- Step 2: mergetime all year batches, then ymonmean ---
+ merged_file = carra2_path / "merged_monthly.nc"
+ if (not check_xr_lazy(merged_file)) or force_overwrite:
+ logger.info("CDO: merging %d year batches...", len(batch_files))
+ cdo.mergetime(
+ input=" ".join(batch_files),
+ output=str(merged_file.resolve()),
+ options=f"-f nc4 -z zip_2 -P {max_workers}",
+ )
+
+ # --- Step 3: multi-year monthly mean + remap ---
+ if (not check_xr_lazy(output_file)) or force_overwrite:
+ logger.info("CDO: computing multi-year monthly mean and remapping to target grid...")
+ ds = cdo.remapycon(
+ str(target_grid_path.resolve()),
+ input=f"-ymonmean {merged_file}",
+ options=f"-f nc4 -z zip_2 -P {max_workers}",
+ returnXDataset=True,
+ )
+
+ # Compute ymonstd for air_temp separately
+ logger.info("CDO: computing multi-year monthly std for air_temp...")
+ ds_std = cdo.remapycon(
+ str(target_grid_path.resolve()),
+ input=f"-selvar,air_temp_sd -ymonmean {merged_file}",
+ options=f"-f nc4 -z zip_2 -P {max_workers}",
+ returnXDataset=True,
+ )
+ ds["air_temp_sd"] = ds_std["air_temp_sd"]
+
+ logger.info("CDO: done")
+
+ ds = ds.drop_vars("time_bnds", errors="ignore")
+
+ month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+ bounds_start = np.cumsum([0] + month_lengths[:-1]).astype("float64")
+ bounds_end = np.cumsum(month_lengths).astype("float64")
+ time_mid = (bounds_start + bounds_end) / 2.0
+
+ time_bounds = np.column_stack([bounds_start, bounds_end])
+
+ ds = ds.assign_coords(time=("time", time_mid))
+ ds["time"].attrs.update(
+ {
+ "standard_name": "time",
+ "units": "days since 0001-01-01",
+ "calendar": "365_day",
+ "bounds": "time_bounds",
+ }
+ )
+ ds["time_bounds"] = (("time", "nv"), time_bounds)
+
+ for var in list(ds.data_vars) + list(ds.coords):
+ ds[var].attrs.pop("missing_value", None)
+ ds[var].attrs.pop("_FillValue", None)
+ ds[var].encoding["missing_value"] = None
+ ds[var].encoding["_FillValue"] = None
+
+ ds.to_netcdf(output_file)
+
+
+def process_hirham(
output_file: str | Path,
- base_url: str,
+ data_dir: str | Path,
vars_dict: dict,
overwrite: bool = False,
max_workers: int = 4,
@@ -128,12 +383,10 @@ def process_hirham_cdo(
Parameters
----------
- data_dir : Union[str, Path]
- Directory containing the input data.
output_file : Union[str, Path]
Path to the output NetCDF file.
- base_url : str
- Base URL for downloading HIRHAM data.
+ data_dir : Union[str, Path]
+ Directory containing the input data.
vars_dict : Dict
Dictionary of variables to process with their attributes.
overwrite : bool, optional
@@ -154,13 +407,7 @@ def process_hirham_cdo(
hirham_zip_dir = hirham_dir / Path("zip")
hirham_zip_dir.mkdir(parents=True, exist_ok=True)
- responses = download_hirham(
- base_url,
- start_year,
- end_year,
- output_dir=hirham_zip_dir,
- max_workers=max_workers,
- )
+ responses = sorted(f for f in hirham_zip_dir.glob("*.zip") if start_year <= int(f.stem) <= end_year)
responses = unzip_files(
responses,
@@ -221,13 +468,36 @@ def _merge_batch(args):
Path to the merged output file.
"""
year, batch, batch_out = args
- cdo_local = Cdo()
- cdo_local.monmean(
- input=f"""-setrtomiss,1e10,1e40 -setrtomiss,-1e40,-1e10 -setmissval,-9e33 -selvar,precipitation,air_temp -setreftime,{start_year}-01-01 -settbounds,day -settaxis,"{year}-01-01" -setattribute,precipitation@standard_name="precipitation_flux" -setattribute,precipitation@units="kg m^-2 day^-1" -aexpr,"precipitation=snowfall+rainfall" -chname,{chname} -setattribute,{setattribute} -selvar,{",".join(vars_dict.keys())} -setgrid,{str(hirham_grid_path.resolve())} -mergetime """
+ tmpdir = tempfile.mkdtemp()
+ cdo_local = Cdo(tempdir=tmpdir)
+ merged = os.path.join(tmpdir, f"merged_{year}.nc")
+ cdo_local.setrtomiss(
+ "-1e40,-1e10",
+ input=f"""-setmissval,-9e33 -selvar,surface_albedo,surface_runoff_flux,surface_melt_flux,climatic_mass_balance,precipitation,air_temp -setreftime,{start_year}-01-01 -settbounds,day -settaxis,"{year}-01-01" -setattribute,precipitation@standard_name="precipitation_flux" -setattribute,precipitation@units="kg m^-2 day^-1" -aexpr,"precipitation=snowfall" -chname,{chname} -setattribute,{setattribute} -selvar,{",".join(vars_dict.keys())} -setgrid,{str(hirham_grid_path.resolve())} -mergetime """
+ " ".join(batch),
+ output=merged,
+ options="-f nc4 -z zip_2 -P 1",
+ )
+ monmean = os.path.join(tmpdir, f"monmean_{year}.nc")
+ cdo_local.monmean(
+ input=merged,
+ output=monmean,
+ options="-f nc4 -z zip_2 -P 1",
+ )
+ monstd = os.path.join(tmpdir, f"monstd_{year}.nc")
+ cdo_local.setattribute(
+ """air_temp_sd@long_name="standard deviation of 2-m air temperature" """,
+ input=f"""-chname,air_temp,air_temp_sd -monstd -selvar,air_temp {merged}""",
+ output=monstd,
+ options="-f nc4 -z zip_2 -P 1",
+ )
+ cdo_local.merge(
+ input=f"{monmean} {monstd}",
output=batch_out,
options="-f nc4 -z zip_2 -P 1",
)
+ shutil.rmtree(tmpdir, ignore_errors=True)
+
return batch_out
batch_files = []
@@ -282,10 +552,9 @@ def _merge_batch(args):
print(f"Time elapsed {time_elapsed:.0f}s")
-def prepare_baseline_climatology(
+def prepare_carra2_climatology(
output_path: Path | str,
- start_year: int,
- end_year: int,
+ year: list[str | int],
version: str,
n_workers: int = 4,
force_overwrite: bool = False,
@@ -297,10 +566,8 @@ def prepare_baseline_climatology(
----------
output_path : Path or str
Output directory.
- start_year : int
- First year of the baseline period.
- end_year : int
- Last year of the baseline period.
+ year : list[str] or list[int]
+ List of years.
version : str
Version string appended to the output filename.
n_workers : int, optional
@@ -316,47 +583,87 @@ def prepare_baseline_climatology(
"""
start_time = time.perf_counter()
- hirham_url = "http://ensemblesrt3.dmi.dk/data/prudence/temp/nichan/Daily2D_GrIS/"
+ output_path = Path(output_path)
+
+ output_file = output_path / Path(f"CARRA2_YMM_{version}.nc")
+ if (not check_xr_lazy(output_file)) or force_overwrite:
+ process_carra2(
+ data_dir=output_path,
+ output_file=output_file,
+ year=year,
+ max_workers=n_workers,
+ )
+
+ elapsed = time.perf_counter() - start_time
+ print(f"Total processing time: {elapsed:.2f} seconds")
+
+ return output_file
+
+
+def prepare_hirham5_climatology(
+ output_file: Path | str,
+ data_path: Path | str,
+ start_year: int,
+ end_year: int,
+ n_workers: int = 4,
+ force_overwrite: bool = False,
+):
+ """
+ Process baseline monthly climatology.
+
+ Parameters
+ ----------
+ output_file : Path or str
+ Output file.
+ data_path : Path or str
+ Data path.
+ start_year : int
+ First year of the baseline period.
+ end_year : int
+ Last year of the baseline period.
+ n_workers : int, optional
+ Number of dask workers, by default 4.
+ force_overwrite : bool, default ``False``
+ If ``True``, downstream helpers may regenerate intermediate/final artifacts
+ even if cache files exist.
+ """
+ start_time = time.perf_counter()
+
hirham_vars_dict: dict[str, dict[str, str]] = {
+ "albedom": {"pism_name": "surface_albedo", "units": ""},
"tas": {"pism_name": "air_temp", "units": "kelvin"},
"gld": {"pism_name": "climatic_mass_balance", "units": "kg m^-2 day^-1"},
+ "rogl": {"pism_name": "surface_runoff_flux", "units": "kg m^-2 day^-1"},
+ "snmel": {"pism_name": "surface_melt_flux", "units": "kg m^-2 day^-1"},
"rainfall": {"pism_name": "rainfall", "units": "kg m^-2 day^-1"},
"snfall": {"pism_name": "snowfall", "units": "kg m^-2 day^-1"},
}
- output_path = Path(output_path)
-
- output_file = output_path / Path(f"HIRHAM5-ERA5_YMM_{start_year}_{end_year}_{version}.nc")
if (not check_xr_lazy(output_file)) or force_overwrite:
- process_hirham_cdo(
- data_dir=output_path,
+ process_hirham(
+ output_file,
+ data_path,
vars_dict=hirham_vars_dict,
start_year=start_year,
end_year=end_year,
- output_file=output_file,
- base_url=hirham_url,
max_workers=n_workers,
)
elapsed = time.perf_counter() - start_time
print(f"Total processing time: {elapsed:.2f} seconds")
- return output_file
-
def prepare_anomalies(
output_path: Path | str,
bucket: str,
prefix: str,
- gcms: list[str],
- present_day_forcings: list[str],
- future_forcings: list[str],
+ gcms: dict,
version: str,
n_workers: int = 4,
force_overwrite: bool = False,
) -> list[Path]:
"""
- Process forcing data for all GCMs and forcings in parallel.
+ Process forcing data for all GCMs and forcings.
Parameters
----------
@@ -366,12 +673,9 @@ def prepare_anomalies(
AWS S3 bucket name containing the forcing data.
prefix : str
S3 key prefix for the forcing data.
- gcms : Sequence[str]
- List of GCM names to process.
- present_day_forcings : list of str
- Present-day forcing experiment names (e.g. ``["pdSST-pdSIC"]``).
- future_forcings : list of str
- Future forcing experiment names (e.g. ``["futSST-pdSIC"]``).
+ gcms : dict[str, list[list[str]]]
+ Mapping of GCM names to their forcing pairs ``[[future, present], ...]``
+ (e.g. ``{"CESM2": [["futSST-pdSIC", "pdSST-pdSIC"]]}``).
version : str
Version string appended to the output filename.
n_workers : int, optional
@@ -397,8 +701,8 @@ def prepare_anomalies(
height_file = forcing_path / Path("height.nc")
start = time.perf_counter()
- # Build list of all (gcm, pd_forcing, ff_forcing) tasks
- tasks = [(gcm, pd_forcing, ff) for gcm in gcms for pd_forcing in present_day_forcings for ff in future_forcings]
+ # Build list of (gcm, pd_forcing, ff_forcing) tasks from per-GCM forcing pairs [[future, present], ...]
+ tasks = [(gcm, pair[1], pair[0]) for gcm, pairs in gcms.items() for pair in pairs]
def _process_anomaly(args):
"""
@@ -415,17 +719,17 @@ def _process_anomaly(args):
Path to the output file.
"""
gcm, pd_forcing, ff = args
- ff_tas_file = forcing_path / Path(ff) / Path(f"tas_Amon_{gcm}_{ff}.nc")
+ ff_tas_file = forcing_path / Path(gcm) / Path(ff) / Path(f"tas_Amon_{gcm}_{ff}.nc")
if ff == "pa-futArcSIC-ext":
- ff_pr_file = forcing_path / Path(ff) / Path(f"pr_day_{gcm}_{ff}.nc")
+ ff_pr_file = forcing_path / Path(gcm) / Path(ff) / Path(f"pr_day_{gcm}_{ff}.nc")
else:
- ff_pr_file = forcing_path / Path(ff) / Path(f"pr_Amon_{gcm}_{ff}.nc")
+ ff_pr_file = forcing_path / Path(gcm) / Path(ff) / Path(f"pr_Amon_{gcm}_{ff}.nc")
if pd_forcing == "pa-pdSIC-ext":
- pd_pr_file = forcing_path / Path(pd_forcing) / Path(f"pr_day_{gcm}_{pd_forcing}.nc")
+ pd_pr_file = forcing_path / Path(gcm) / Path(pd_forcing) / Path(f"pr_day_{gcm}_{pd_forcing}.nc")
else:
- pd_pr_file = forcing_path / Path(pd_forcing) / Path(f"pr_Amon_{gcm}_{pd_forcing}.nc")
+ pd_pr_file = forcing_path / Path(gcm) / Path(pd_forcing) / Path(f"pr_Amon_{gcm}_{pd_forcing}.nc")
- pd_tas_file = forcing_path / Path(pd_forcing) / Path(f"tas_Amon_{gcm}_{pd_forcing}.nc")
+ pd_tas_file = forcing_path / Path(gcm) / Path(pd_forcing) / Path(f"tas_Amon_{gcm}_{pd_forcing}.nc")
output_file = forcing_path / Path(f"{gcm}_anomalies_{ff}_{pd_forcing}_{version}.nc")
if (not check_xr_lazy(output_file, verbose=False)) or force_overwrite:
@@ -433,11 +737,17 @@ def _process_anomaly(args):
cdo_local = Cdo(tempdir=tmpdir)
ds = cdo_local.setmisstodis(
- input=f"""-remapycon,{str(target_grid_path.resolve())} -chname,pr,precipitation,tas,air_temp -merge -setattribute,height@units="m",height@standard_name="surface_altitude" -selvar,height {height_file} -sub -merge [ -selvar,tas {str(ff_tas_file.resolve())} -selvar,pr {str(ff_pr_file.resolve())} ] -merge [ -selvar,tas {str(pd_tas_file.resolve())} -selvar,pr {str(pd_pr_file.resolve())} ] """,
+ input=f"""-remapycon,{str(target_grid_path.resolve())} -chname,pr,precipitation,tas,air_temp -merge -setattribute,height@units="m",height@standard_name="surface_altitude" -selvar,height {height_file} -sub -merge [ -selvar,tas {str(ff_tas_file.resolve())} -maxc,0 -selvar,pr {str(ff_pr_file.resolve())} ] -merge [ -selvar,tas {str(pd_tas_file.resolve())} -maxc,0 -selvar,pr {str(pd_pr_file.resolve())} ] """,
returnXDataset=True,
options="-f nc4 -z zip_2 -P 1",
)
+ # Replace inf values introduced by CDO's setmisstodis interpolation
+ for v in list(ds.data_vars) + list(ds.coords):
+ if np.issubdtype(ds[v].dtype, np.floating):
+ ds[v] = ds[v].where(np.isfinite(ds[v]), 0.0)
+
+ ds = ds.drop_vars("height", errors="ignore").drop_dims("height", errors="ignore")
ds["air_temp"].attrs["units"] = "kelvin"
ds["precipitation"] = ds["precipitation"] * 86400.0
ds["precipitation"].attrs["units"] = "kg m^-2 day^-1"
@@ -494,8 +804,7 @@ def baseline_with_anomalies(
Add baseline climatology to each anomaly forcing file.
For precipitation and air_temp the baseline values are added to the
- anomaly. The ``height`` variable is taken from the anomaly file
- unchanged. Output files are written next to the baseline file with
+ anomaly. Output files are written next to the baseline file with
a combined name.
Parameters
@@ -532,7 +841,6 @@ def baseline_with_anomalies(
ds = baseline.copy(deep=True)
ds["precipitation"] = baseline["precipitation"] + anomaly["precipitation"]
ds["air_temp"] = baseline["air_temp"] + anomaly["air_temp"]
- ds["height"] = anomaly["height"]
for var in list(ds.data_vars) + list(ds.coords):
ds[var].attrs.pop("missing_value", None)
@@ -547,3 +855,118 @@ def baseline_with_anomalies(
result.append(output_file)
return result
+
+
+ICE_DENSITY = 910.0 # kg m^-3
+
+
+def prepare_ocean_forcing(
+ input_path: str | Path,
+ output_path: str | Path,
+ bmelt_0: float = 228.0,
+ bmelt_1: float = 10.0,
+ lat_0: float = 69.0,
+ lat_1: float = 80.0,
+ process_mask: bool = False,
+ crs: str = "EPSG:3413",
+) -> None:
+ """
+ Write latitude-dependent ocean forcing.
+
+ Parameters
+ ----------
+ input_path : str or Path
+ Path to the input NetCDF file.
+ output_path : str or Path
+ Path to the output NetCDF file.
+ bmelt_0 : float
+ Southern basal melt rate in m yr-1.
+ bmelt_1 : float
+ Northern basal melt rate in m yr-1.
+ lat_0 : float
+ Latitude for the southern melt rate.
+ lat_1 : float
+ Latitude for the northern melt rate.
+ process_mask : bool
+ If True, zero out melt on non-ocean cells.
+ crs : str
+ Coordinate reference system string.
+ """
+
+ input_path = Path(input_path)
+ output_path = Path(output_path)
+ ds = xr.open_dataset(input_path)
+ proj = Proj(crs)
+
+ x = ds["x"].values
+ y = ds["y"].values
+ X, Y = np.meshgrid(x, y)
+ _, Lat = proj(X, Y, inverse=True)
+
+ # Convert melt rates: m yr-1 -> kg m-2 yr-1
+ bmelt_0_kg = bmelt_0 * ICE_DENSITY
+ bmelt_1_kg = bmelt_1 * ICE_DENSITY
+
+ # Linear interpolation: bmelt = a * lat + b
+ a = (bmelt_1_kg - bmelt_0_kg) / (lat_1 - lat_0)
+ b_intercept = bmelt_0_kg - a * lat_0
+
+ bmelt = a * Lat + b_intercept
+ bmelt = np.clip(bmelt, min(bmelt_0_kg, bmelt_1_kg), max(bmelt_0_kg, bmelt_1_kg))
+
+ if process_mask and "mask" in ds:
+ land_mask = (ds["mask"].values != 0) & (ds["mask"].values != 3)
+ if land_mask.ndim == 3:
+ land_mask = land_mask[0]
+ bmelt[land_mask] = 0.0
+
+ ds.close()
+
+ # Build output dataset
+ time_mid = np.array([187.5])
+ time_bnds = np.array([[0.0, 365.0]])
+
+ time_coord = xr.Variable(
+ "time",
+ time_mid,
+ attrs={
+ "units": "days since 0001-01-01",
+ "calendar": "365_day",
+ "standard_name": "time",
+ "axis": "T",
+ "bounds": "time_bnds",
+ },
+ )
+
+ out = xr.Dataset(
+ {
+ "shelfbmassflux": xr.Variable(
+ ("time", "y", "x"),
+ bmelt[np.newaxis, :, :],
+ attrs={"units": "kg m-2 yr-1", "grid_mapping": "mapping"},
+ ),
+ "shelfbtemp": xr.Variable(
+ ("time", "y", "x"),
+ np.zeros((1, len(y), len(x))),
+ attrs={"units": "deg_C", "grid_mapping": "mapping"},
+ ),
+ "time_bnds": xr.Variable(("time", "nb2"), time_bnds),
+ },
+ coords={
+ "time": time_coord,
+ "x": xr.Variable("x", x, attrs={"units": "m", "standard_name": "projection_x_coordinate", "axis": "X"}),
+ "y": xr.Variable("y", y, attrs={"units": "m", "standard_name": "projection_y_coordinate", "axis": "Y"}),
+ },
+ )
+ out.rio.write_crs(crs, inplace=True)
+
+ for var in list(ds.data_vars) + list(ds.coords):
+ ds[var].attrs.pop("missing_value", None)
+ ds[var].attrs.pop("_FillValue", None)
+ ds[var].encoding["missing_value"] = None
+ ds[var].encoding["_FillValue"] = None
+
+ encoding: dict[str, dict[str, Any]] = {var: {"_FillValue": None} for var in list(out.data_vars) + list(out.coords)}
+ encoding["shelfbmassflux"].update({"zlib": True, "complevel": 2})
+ encoding["shelfbtemp"].update({"zlib": True, "complevel": 2})
+ out.to_netcdf(output_path, encoding=encoding, engine="h5netcdf")
diff --git a/pism_terra/kitp/postprocess.py b/pism_terra/kitp/postprocess.py
index e199c47..2405134 100644
--- a/pism_terra/kitp/postprocess.py
+++ b/pism_terra/kitp/postprocess.py
@@ -22,6 +22,7 @@
"""
import json
+import logging
import time
import warnings
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
@@ -36,8 +37,13 @@
from pyfiglet import Figlet
from tqdm import tqdm
+from pism_terra.log import setup_logging
+
xr.set_options(keep_attrs=True)
warnings.filterwarnings("ignore", message="invalid value encountered in cast", category=RuntimeWarning)
+warnings.filterwarnings("ignore", message="pkg_resources is deprecated", category=UserWarning)
+
+logger = logging.getLogger(__name__)
def process_file(
@@ -47,8 +53,10 @@ def process_file(
Clip a NetCDF dataset to the glacier geometry defined in an BASIN file.
This function reads a NetCDF file containing geospatial data and clips it to the
- geometry defined in a glacier outline file (e.g., BASIN shapefile). The clipped dataset
- is saved to a new NetCDF file prefixed with "clipped_".
+ geometry defined in a glacier outline file (e.g., BASIN shapefile). Clipped spatial
+ output is written to ``/processed_spatial/clipped_.nc`` and
+ per-basin scalar sums to ``/processed_scalar/fldsum_.nc``,
+ where ``output_root`` is the parent of the input file's directory.
Parameters
----------
@@ -56,89 +64,95 @@ def process_file(
Path to the NetCDF file to be clipped. Must contain x/y spatial dimensions.
basin_file : str or Path
Path to the BASIN glacier outline file (e.g., GeoPackage or shapefile) that defines
- the geometry to clip the dataset to. Must include an `epsg` column to define the CRS.
+ the geometry to clip the dataset to.
client : dask.Client
Dask client.
- column : str
- Column.
- crs : str
- CRS code.
+ column : str, default "SUBREGION1"
+ Name of the column in ``basin_file`` used to identify basins (e.g.
+ ``"GIS"`` is selected for the merged-basin clip).
+ crs : str, default "EPSG:3413"
+ CRS code applied to the input dataset before clipping.
"""
infile = Path(infile)
infile_name = infile.name
- infile_path = infile.parent
- clipped_file = infile_path / Path("clipped_" + infile_name)
- scalar_file = infile_path / Path("fldsum_" + infile_name)
+ output_root = infile.parent.parent
+ clipped_dir = output_root / "processed_spatial"
+ scalar_dir = output_root / "processed_scalar"
+ clipped_dir.mkdir(parents=True, exist_ok=True)
+ scalar_dir.mkdir(parents=True, exist_ok=True)
+ clipped_file = clipped_dir / Path("clipped_" + infile_name)
+ scalar_file = scalar_dir / Path("fldsum_" + infile_name)
basin = gpd.read_file(basin_file)
+ if basin.crs is None or str(basin.crs) != str(crs):
+ basin = basin.to_crs(crs)
start = time.time()
-
- ds = (
- xr.open_dataset(
- infile,
- decode_times=False,
- decode_timedelta=False,
- chunks="auto",
- engine="h5netcdf",
- )
- .drop_vars("time_bounds", errors="ignore")
- .rio.set_spatial_dims(x_dim="x", y_dim="y")
+ time_coder = xr.coders.CFDatetimeCoder(use_cftime=False)
+
+ ds = xr.open_dataset(
+ infile,
+ decode_timedelta=False,
+ decode_times=False,
+ chunks="auto",
+ engine="netcdf4",
)
- ds = ds.rio.write_crs(crs, inplace=False)
+ # Spatial bounds vars (``x_bnds``/``y_bnds``) reference the pre-clip
+ # x/y sizes and would inject dangling dimensions back into the output
+ # after merge — h5netcdf serializes those as duplicate "x" dims. Drop
+ # them before splitting; PISM doesn't require them on the clipped output.
+ ds = ds.drop_vars(["x_bnds", "x_bounds", "y_bnds", "y_bounds", "mapping"], errors="ignore")
- # Separate variables that lack spatial (x, y) dimensions, as rio.clip cannot handle them
- non_spatial_vars = [var for var in ds.data_vars if "x" not in ds[var].dims or "y" not in ds[var].dims]
+ # Separate variables that lack BOTH spatial (x, y) dimensions, as
+ # rio.clip cannot handle them. Use ``and`` so that vars carrying only
+ # one spatial dim (rare, but possible) still go down the spatial path.
+ non_spatial_vars = [var for var in ds.data_vars if "x" not in ds[var].dims and "y" not in ds[var].dims]
ds_non_spatial = ds[non_spatial_vars]
- ds = ds.drop_vars(non_spatial_vars)
-
+ ds = ds.drop_vars(non_spatial_vars).rio.write_crs(crs).rio.set_spatial_dims(x_dim="x", y_dim="y")
ds = client.persist(ds)
progress(ds)
- gis_clipped = ds.rio.clip(basin[basin[column] == "GIS"].geometry, drop=False)
- gis_clipped = xr.merge([gis_clipped, ds_non_spatial])
- print(f"Writing {clipped_file}")
comp = {"zlib": True, "complevel": 2}
- encoding = {var: comp for var in gis_clipped.data_vars}
- write_clipped = gis_clipped.to_netcdf(clipped_file, engine="h5netcdf", encoding=encoding, compute=False)
- future_clipped = client.compute(write_clipped)
- progress(future_clipped)
dss = []
- for _, basin in tqdm(basin.iterrows(), total=len(basin), desc="Clipping basins"):
- ds_clipped = ds.rio.clip([basin.geometry], drop=False)
- dss.append(ds_clipped.expand_dims({"basin": [basin[column]]}))
-
- clipped = xr.concat(dss, dim="basin")
-
- print(f"Writing {scalar_file}")
- scalar = clipped.sum(dim=["y", "x"])
+ for _, row in tqdm(basin.iterrows(), total=len(basin), desc="Clipping basins"):
+ ds_clipped = ds.rio.clip([row.geometry], drop=False)
+ ds_sum = ds_clipped.sum(dim=["y", "x"]).compute()
+ ds_sum["area"] = row.geometry.area
+ ds_sum["area"].attrs.update({"units": "m^2"})
+ dss.append(ds_sum.expand_dims({"basin": [row[column]]}))
+
+ scalar = xr.concat(dss, dim="basin")
+
+ logger.info("Writing %s", scalar_file)
+ # Keep non-spatial vars (e.g. pism_config)
+ extra_vars = [v for v in ds_non_spatial.data_vars if "time" not in ds_non_spatial[v].dims]
+ if extra_vars:
+ scalar = xr.merge([scalar, ds_non_spatial[extra_vars].compute()])
encoding_scalar = {var: comp for var in scalar.data_vars}
- write_scalar = scalar.to_netcdf(scalar_file, engine="h5netcdf", encoding=encoding_scalar, compute=False)
- future_scalar = client.compute(write_scalar)
- progress(future_scalar)
+ scalar.to_netcdf(scalar_file, encoding=encoding_scalar, engine="netcdf4")
end = time.time()
time_elapsed = end - start
- print(f"Time elapsed for {infile_name}: {time_elapsed:.0f}s")
+ logger.info("Time elapsed for %s: %.0fs", infile_name, time_elapsed)
-def postprocess_glacier(config_file: str | Path):
+def postprocess_glacier(config_file: str | Path, n_workers: int = 4):
"""
- Configure and print a PISM model run command for a glacier.
+ Postprocess KITP output by clipping spatial output to basin geometries.
- This function reads glacier metadata from a CSV file and simulation settings
- from a TOML configuration file, then builds and prints a full PISM command-line
- string for executing a model run. It sets up output directories and constructs
- appropriate output filenames.
+ Reads a TOML run-configuration, opens the configured ``spatial`` output
+ NetCDF, and clips it to the basin outline using a Dask client.
Parameters
----------
config_file : str or Path
- Path to a TOML file containing PISM run configuration, including time,
- energy model, stress balance model, and reporting options.
+ Path to a TOML file containing PISM run configuration with at least
+ ``[basin].outline`` and ``[output].spatial`` keys.
+ n_workers : int, optional
+ Number of Dask workers, by default 4.
"""
config_toml = toml.load(config_file)
@@ -147,8 +161,8 @@ def postprocess_glacier(config_file: str | Path):
start = time.time()
outline_file = config["basin"]["outline"]
- client = Client(n_workers=4, threads_per_worker=1, memory_limit="8GiB")
- print(f"Dask dashboard: {client.dashboard_link}")
+ client = Client(n_workers=n_workers, threads_per_worker=1)
+ logger.info("Dask dashboard: %s", client.dashboard_link)
for o in ["spatial"]:
s_file = Path(config["output"][o])
@@ -158,7 +172,7 @@ def postprocess_glacier(config_file: str | Path):
end = time.time()
time_elapsed = end - start
- print(f"Time elapsed {time_elapsed:.0f}s")
+ logger.info("Time elapsed %.0fs", time_elapsed)
def main():
@@ -169,6 +183,12 @@ def main():
# set up the option parser
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.description = "Postprocess KITP Greenland."
+ parser.add_argument(
+ "--ntasks",
+ help="Sets number of tasks.",
+ type=int,
+ default=4,
+ )
parser.add_argument(
"RUN_FILE",
help="CONFIG TOML.",
@@ -177,8 +197,12 @@ def main():
options, unknown = parser.parse_known_args()
config_file = options.RUN_FILE[0]
+ ntasks = options.ntasks
+
+ config_path = Path(config_file).resolve().parent
+ setup_logging(config_path / "postprocess.log")
- postprocess_glacier(config_file)
+ postprocess_glacier(config_file, n_workers=ntasks)
if __name__ == "__main__":
diff --git a/pism_terra/kitp/prepare.py b/pism_terra/kitp/prepare.py
index d8b5deb..e477615 100644
--- a/pism_terra/kitp/prepare.py
+++ b/pism_terra/kitp/prepare.py
@@ -21,6 +21,7 @@
Prepare ISMIP7 Greenland data sets.
"""
+import logging
import os
import re
import shutil
@@ -47,14 +48,22 @@
from pism_terra.kitp.forcing import (
baseline_with_anomalies,
prepare_anomalies,
- prepare_baseline_climatology,
+ prepare_carra2_climatology,
+ prepare_hirham5_climatology,
+ prepare_ocean_forcing,
)
+from pism_terra.prepare_select import add_include_argument, select_datasets
from pism_terra.raster import create_ds
from pism_terra.vector import dissolve
from pism_terra.workflow import check_xr_fully, check_xr_lazy
xr.set_options(keep_attrs=True)
+logger = logging.getLogger(__name__)
+
+# Datasets the KITP Greenland prepare can process, in execution order.
+KITP_DATASETS = ["grid", "observations", "ocean", "baseline", "anomalies"]
+
def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
"""
@@ -77,12 +86,15 @@ def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
dict[str, Any]
Results dictionary containing:
- - ``"config"``: dict
- The parsed TOML configuration used for processing.
+ - ``"config"`` : dict — parsed TOML configuration.
+ - ``"grid_file"`` : Path — generated grid NetCDF.
+ - ``"boot_file"`` : Path — observation-derived boot NetCDF.
+ - ``"heatflux_file"`` : Path — geothermal heat-flux NetCDF.
+ - ``"baseline_file"`` : Path — baseline climatology NetCDF.
"""
parser = ArgumentParser()
- parser.add_argument("--obs-path", default="data/obs")
+ parser.add_argument("--data-path", default="data")
parser.add_argument(
"--force-overwrite",
help="Force downloading all files.",
@@ -95,6 +107,7 @@ def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
type=int,
default=8,
)
+ add_include_argument(parser, KITP_DATASETS)
parser.add_argument("CONFIG_FILE", nargs=1)
parser.add_argument("OUTPUT_PATH", nargs=1)
args = parser.parse_args(list(argv) if argv is not None else None)
@@ -102,26 +115,37 @@ def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
config_file = args.CONFIG_FILE[0]
force_overwrite = args.force_overwrite
ntasks = args.ntasks
- obs_path = Path(args.obs_path)
+ data_path = Path(args.data_path)
+ selected = select_datasets(args.include, KITP_DATASETS)
+
output_path = Path(args.OUTPUT_PATH[0])
output_path.mkdir(parents=True, exist_ok=True)
+ log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+ logging.basicConfig(level=logging.WARNING, format=log_format)
+ for handler in logging.root.handlers:
+ handler.setLevel(logging.WARNING)
+ file_handler = logging.FileHandler(output_path / "prepare.log")
+ file_handler.setLevel(logging.INFO)
+ file_handler.setFormatter(logging.Formatter(log_format))
+ logging.getLogger("pism_terra").setLevel(logging.INFO)
+ logging.getLogger("pism_terra").addHandler(file_handler)
+
f = Figlet(font="standard")
banner = f.renderText("pism-terra")
- print("=" * 120)
- print(banner)
- print("=" * 120)
- print("Preparing ISMIP7 Greenland data")
- print("-" * 120)
- print("")
+ logger.info("=" * 120)
+ logger.info("\n%s", banner)
+ logger.info("=" * 120)
+ logger.info("Preparing ISMIP7 Greenland data")
+ logger.info("-" * 120)
config = toml.loads(Path(config_file).read_text("utf-8"))
version = config["version"]
- print("-" * 120)
- print("Grid File")
- print("-" * 120)
+ stage_path = output_path / Path(f"stage_{config["version"]}")
+ stage_path.mkdir(exist_ok=True)
+ crs = config["domain"]["crs"]
x_bnds = config["domain"]["x_bounds"]
y_bnds = config["domain"]["y_bounds"]
resolution_str = config["domain"]["resolution"]
@@ -130,82 +154,146 @@ def main(argv: Sequence[str] | None = None) -> dict[str, Any]:
raise ValueError(f"Cannot parse resolution string: {resolution_str!r}")
resolution, _ = int(match.group(1)), match.group(2)
- grid_ds = create_domain(x_bnds, y_bnds, resolution)
- grid_file = output_path / Path("ismip7_greenland_grid.nc")
- encoding = {var: {"_FillValue": None} for var in list(grid_ds.data_vars) + list(grid_ds.coords)}
- grid_ds.to_netcdf(grid_file, encoding=encoding)
- check_xr_fully(grid_file)
-
- url = "https://g-ab4495.8c185.08cc.data.globus.org/ISMIP6/ISMIP7_Prep/Observations/Greenland/GreenlandObsISMIP7-v1.3.nc"
- print("-" * 120)
- print("Boot File")
- print("-" * 120)
- obs_files = prepare_observations(
- url,
- obs_path,
- output_path,
- config,
- target_grid=grid_ds,
- force_overwrite=force_overwrite,
- )
- for v in obs_files.values():
- check_xr_lazy(v)
-
- print("-" * 120)
- print("Baseline Climatology")
- print("-" * 120)
- start_year = config["pathway"]["baseline"]["start_year"]
- end_year = config["pathway"]["baseline"]["end_year"]
- baseline_file = prepare_baseline_climatology(
- output_path,
- start_year=start_year,
- end_year=end_year,
- version=version,
- n_workers=ntasks,
- force_overwrite=force_overwrite,
- )
-
- print("-" * 120)
- print("Anomaly Forcing")
- print("-" * 120)
-
- bucket = config["forcing"]["bucket"]
- prefix = config["forcing"]["prefix"]
- gcms = config["gcms"]
- present_day_forcings = config["forcing"]["present_day_forcings"]
- future_forcings = config["forcing"]["future_forcings"]
-
- forcing_files = prepare_anomalies(
- output_path,
- bucket=bucket,
- prefix=prefix,
- gcms=gcms,
- present_day_forcings=present_day_forcings,
- future_forcings=future_forcings,
- version=version,
- n_workers=ntasks,
- force_overwrite=force_overwrite,
- )
-
- combined_files = baseline_with_anomalies(baseline_file, forcing_files)
-
- input_files = [grid_file] + list(obs_files.values()) + [baseline_file] + forcing_files + combined_files
-
- s3_output_path = output_path / Path(config["prefix"])
+ # --- Grid (a dependency of the observations target grid) ---
+ grid_file = data_path / Path("ismip7_greenland_grid.nc")
+ grid_ds = None
+ if {"grid", "observations"} & set(selected):
+ logger.info("-" * 120)
+ logger.info("Grid File")
+ logger.info("-" * 120)
+ grid_ds = create_domain(x_bnds, y_bnds, resolution, crs=crs)
+ if "grid" in selected:
+ grid_ds.to_netcdf(grid_file)
+ check_xr_fully(grid_file)
+
+ # --- Observations (boot, heatflux) ---
+ obs_files: dict[str, Any] = {}
+ if "observations" in selected:
+ url = "https://g-ab4495.8c185.08cc.data.globus.org/ISMIP6/ISMIP7_Prep/Observations/Greenland/GreenlandObsISMIP7-v1.3.nc"
+ logger.info("-" * 120)
+ logger.info("Boot File")
+ logger.info("-" * 120)
+ obs_path = data_path / Path("obs")
+ obs_files = prepare_observations(
+ url,
+ obs_path,
+ stage_path,
+ config,
+ target_grid=grid_ds,
+ force_overwrite=force_overwrite,
+ )
+ for v in obs_files.values():
+ check_xr_lazy(v)
+
+ # --- Ocean forcing (needs the boot file from observations) ---
+ ocean_forcing_file = None
+ if "ocean" in selected:
+ if "boot_file" not in obs_files:
+ raise SystemExit("--include ocean requires 'observations' (it reads the boot file).")
+ logger.info("-" * 120)
+ logger.info("Ocean Forcing")
+ logger.info("-" * 120)
+
+ bmelt_0: float = 228.0
+ bmelt_1: float = 10.0
+ lat_0: float = 69.0
+ lat_1: float = 80.0
+
+ ocean_forcing_file = stage_path / Path(f"ocean_forcing_{bmelt_0}_{bmelt_1}_{lat_0}_{lat_1}.nc")
+ prepare_ocean_forcing(
+ input_path=obs_files["boot_file"],
+ output_path=ocean_forcing_file,
+ bmelt_0=bmelt_0,
+ bmelt_1=bmelt_1,
+ lat_0=lat_0,
+ lat_1=lat_1,
+ )
+
+ # --- Baseline climatology ---
+ baseline_file = None
+ if "baseline" in selected:
+ logger.info("-" * 120)
+ logger.info("Baseline Climatology")
+ logger.info("-" * 120)
+ baseline = config["baseline"]
+ if baseline == "hirham5":
+ start_year = config["climatology"][baseline]["start_year"]
+ end_year = config["climatology"][baseline]["end_year"]
+ baseline_file = stage_path / Path(f"HIRHAM5-ERA5_YMM_{start_year}_{end_year}_{version}.nc")
+ prepare_hirham5_climatology(
+ baseline_file,
+ data_path,
+ start_year=start_year,
+ end_year=end_year,
+ n_workers=ntasks,
+ force_overwrite=force_overwrite,
+ )
+ elif baseline == "carra2":
+ year = config["climatology"][baseline]["year"]
+ baseline_file = prepare_carra2_climatology(
+ data_path,
+ year=year,
+ version=version,
+ n_workers=ntasks,
+ force_overwrite=force_overwrite,
+ )
+ else:
+ raise ValueError(f"Unknown baseline {baseline!r}. Supported: 'hirham5', 'carra2'.")
+
+ # --- Anomaly forcing (combined with the baseline when both are present) ---
+ forcing_files: list = []
+ combined_files: list = []
+ if "anomalies" in selected:
+ logger.info("-" * 120)
+ logger.info("Anomaly Forcing")
+ logger.info("-" * 120)
+
+ bucket = config["forcing"]["bucket"]
+ prefix = config["forcing"]["prefix"]
+ gcms = config["gcms"]
+
+ forcing_files = list(
+ prepare_anomalies(
+ stage_path,
+ bucket=bucket,
+ prefix=prefix,
+ gcms=gcms,
+ version=version,
+ n_workers=ntasks,
+ force_overwrite=force_overwrite,
+ )
+ )
+ if baseline_file is not None:
+ combined_files = list(baseline_with_anomalies(baseline_file, forcing_files))
+ else:
+ logger.warning("Anomalies produced without a baseline; skipping baseline_with_anomalies merge.")
+
+ # Ship only the outputs produced by the selected datasets.
+ input_files: list = []
+ if "grid" in selected:
+ input_files.append(grid_file)
+ input_files += list(obs_files.values())
+ if baseline_file is not None:
+ input_files.append(baseline_file)
+ if ocean_forcing_file is not None:
+ input_files.append(ocean_forcing_file)
+ input_files += combined_files
+
+ s3_output_path = output_path / Path(config["prefix"]) / Path(config["version"])
s3_output_path.mkdir(parents=True, exist_ok=True)
- print("-" * 120)
- print(f"Copying input files to {s3_output_path}")
- print("-" * 120)
+ logger.info("-" * 120)
+ logger.info("Copying input files to %s", s3_output_path)
+ logger.info("-" * 120)
for f in input_files:
dest = s3_output_path / Path(f).name
shutil.copy2(f, dest)
- print(f" {dest}")
+ logger.info(" %s", dest)
return {
"config": config,
"grid_file": grid_file,
- "boot_file": obs_files["boot_file"],
- "heatflux_file": obs_files["heatflux_file"],
+ "boot_file": obs_files.get("boot_file"),
+ "heatflux_file": obs_files.get("heatflux_file"),
"baseline_file": baseline_file,
}
diff --git a/pism_terra/kitp/run.py b/pism_terra/kitp/run.py
index 34ae960..b05658f 100644
--- a/pism_terra/kitp/run.py
+++ b/pism_terra/kitp/run.py
@@ -34,15 +34,16 @@
from jinja2 import Environment, FileSystemLoader, StrictUndefined
from pyfiglet import Figlet
-from pism_terra.config import JobConfig, RunConfig, load_config, load_uq
+from pism_terra.config import JobConfig, load_config, load_uq
from pism_terra.kitp.stage import stage
-from pism_terra.sampling import create_samples
+from pism_terra.sampling import generate_samples
from pism_terra.workflow import (
apply_choice_mapping,
dict2str,
- merge_model,
+ filter_overrides_by_config,
normalize_row,
sort_dict_by_key,
+ validate_pism_options,
)
# one Jinja environment for all renders
@@ -52,17 +53,14 @@
def run_kitp(
config_file: str | Path,
template_file: Path | str,
- outline_file: Path | str,
+ outline_file: Path | str | None,
path: str | Path = "result",
- resolution: None | str = None,
- nodes: None | int = None,
- ntasks: None | int = None,
- queue: None | str = None,
- walltime: None | str = None,
+ config_cli: dict | None = None,
debug: bool = False,
*,
uq: Mapping[str, object] | pd.Series | None = None,
sample: int | None = None,
+ pism_config_cdl: str | Path | None = None,
):
"""
Configure and generate a PISM job script for a single glacier (ensemble-ready).
@@ -84,21 +82,16 @@ def run_kitp(
outline_file : str or pathlib.Path
Path to a geopandas file with the glacier outline.
path : str or pathlib.Path, optional
- Base output directory. A subfolder ``/`` is created with
- ``output/`` and ``run_scripts/`` subdirectories. Default is ``"result"``.
- resolution : str or None, optional
- Grid resolution (e.g., ``"200m"``). If ``None``, the value from
- ``[grid].resolution`` in the config is used.
- nodes : int or None, optional
- Node count override for the submission template. If ``None``, use config.
- ntasks : int or None, optional
- MPI task count override for the submission template/run options.
- If ``None``, use config.
- queue : str or None, optional
- Batch queue/partition override for the submission template. If ``None``,
- use config.
- walltime : str or None, optional
- Wall time override in ``HH:MM:SS``. If ``None``, use config.
+ Base output directory. ``output/`` and ``run_scripts/`` subdirectories
+ are created inside it. Default is ``"result"``.
+ config_cli : dict or None, optional
+ CLI-side overrides applied after reading the config. Recognized keys:
+ ``"resolution"`` (e.g. ``"200m"``), ``"nodes"`` (int), ``"ntasks"``
+ (int), ``"tasks"`` (int, MPI tasks per node), ``"queue"`` (str),
+ ``"walltime"`` (``HH:MM:SS``), ``"stress_balance"`` (sub-model name
+ swap, e.g. ``"sia"``), and ``"start"`` / ``"end"`` (``YYYY-MM-DD``
+ time bounds). Any value of ``None`` falls back to the config file.
+ Default is ``None`` (no overrides).
debug : bool, optional
If ``True``, skip rendering the template (leave it empty) but still
append the constructed PISM command line to the output script.
@@ -114,6 +107,9 @@ def run_kitp(
``"sample"``, that value is used. The value changes the filename
stem used for outputs (e.g., ``..._s0042``). If neither is provided,
filenames use a descriptive ``surface/energy/stress_balance`` suffix.
+ pism_config_cdl : str or Path or None, optional
+ Path to a PISM CDL master config file. If provided, all run options
+ are validated against it before generating the command line.
Raises
------
@@ -136,29 +132,30 @@ def run_kitp(
--------
Basic use with config and template:
- >>> run_glacier(
- ... rgi_id="RGI2000-v7.0-C-01-04374",
+ >>> run_kitp(
... config_file="config/init_stampede3.toml",
... template_file="templates/stampede3.j2",
+ ... outline_file=None,
... path="result",
... )
Ensemble member with overrides from a pandas row (e.g., Latin Hypercube):
>>> row = df_samples.loc[17] # contains dotted keys + 'sample'
- >>> run_glacier(
- ... rgi_id="RGI2000-v7.0-C-01-04374",
+ >>> run_kitp(
... config_file="config/init_stampede3.toml",
... template_file="templates/stampede3.j2",
+ ... outline_file=None,
... uq=row, # dotted PISM flags to override
... sample=None, # will be inferred from row['sample'] if present
... ntasks=112, # optional template/run override
... )
"""
- outline_file = Path(outline_file)
cfg = load_config(config_file)
+ config_cli = config_cli or {}
+ resolution = config_cli.get("resolution")
if resolution:
resolution = re.sub(r"\s+", "", resolution)
@@ -183,7 +180,6 @@ def run_kitp(
run = {}
for section in (
"geometry",
- "ocean",
"calving",
"iceflow",
"reporting",
@@ -192,6 +188,7 @@ def run_kitp(
):
run.update(getattr(cfg, section))
run.update(cfg.atmosphere.selected())
+ run.update(cfg.ocean.selected())
run.update(cfg.energy.selected())
run.update(cfg.frontal_melt.selected())
run.update(cfg.grid.as_params())
@@ -207,11 +204,37 @@ def run_kitp(
start = cfg.model_dump(by_alias=True)["time"]["time.start"]
end = cfg.model_dump(by_alias=True)["time"]["time.end"]
- writer = cfg.model_dump()["run"]["writer"] if (cfg.model_dump()["run"]["writer"] is not None) else ""
if resolution is None:
resolution = cfg.model_dump(by_alias=True)["grid"]["resolution"]
- stress_balance = cfg.model_dump(by_alias=True)["stress_balance"]["model"]
+ stress_balance = config_cli.get("stress_balance")
+ if stress_balance is None:
+ stress_balance = cfg.model_dump(by_alias=True)["stress_balance"]["model"]
+ else:
+ # Swap the selected stress-balance model. Drop the previous model's
+ # options from ``run`` first so leftover keys from e.g. blatter don't
+ # leak into a sia run.
+ for old_key in cfg.stress_balance.selected():
+ run.pop(old_key, None)
+ cfg.stress_balance.model = stress_balance
+ run.update(cfg.stress_balance.selected())
+
+ # CLI overrides for time bounds. ``cfg.time`` is a TimeConfig pydantic
+ # model with field names ``time_start`` / ``time_end`` (aliased to the
+ # dotted ``"time.start"`` / ``"time.end"``), so attribute assignment is
+ # required. Drop the prior dotted entry from ``run`` and re-apply via
+ # ``as_params()`` so the new value lands cleanly.
+ _start = config_cli.get("start")
+ _end = config_cli.get("end")
+ if _start is not None:
+ run.pop("time.start", None)
+ cfg.time.time_start = _start
+ run.update(cfg.time.as_params())
+ if _end is not None:
+ run.pop("time.end", None)
+ cfg.time.time_end = _end
+ run.update(cfg.time.as_params())
+
energy = cfg.model_dump(by_alias=True)["energy"]["model"]
surface = cfg.model_dump(by_alias=True)["surface"]["model"]
@@ -219,7 +242,7 @@ def run_kitp(
name_options = f"surface_{surface}_energy_{energy}_stress_balance_{stress_balance}"
else:
name_options = f"id_{sample}"
- run.update({"output.experiment_id": sample})
+ # run.update({"output.experiment_id": sample})
uq_clean = normalize_row(uq) if uq is not None else {}
# Prefer explicit `sample` arg; else default from uq['sample']
@@ -229,8 +252,12 @@ def run_kitp(
except Exception:
pass
- # Remove 'sample' from flag overrides
+ # Remove 'sample' from flag overrides; drop any key not in the config-derived
+ # run dict (e.g., surface.debm_simple.std_dev.file when surface.model == "pdd").
overrides = {k: v for k, v in uq_clean.items() if k != "sample"}
+ overrides, skipped = filter_overrides_by_config(overrides, run.keys())
+ if skipped:
+ print(f"Skipping uq overrides not in config: {skipped}")
# Apply to runtime dict (these should be dotted PISM flags)
run.update(overrides)
@@ -245,34 +272,35 @@ def run_kitp(
}
)
- run_str = dict2str(sort_dict_by_key(run)) + f" {writer}"
+ if pism_config_cdl is not None:
+ validate_pism_options(run, pism_config_cdl)
+
+ run_str = dict2str(sort_dict_by_key(run))
- run_opts = RunConfig(**cfg.run.model_dump())
job_opts = JobConfig(**cfg.job.model_dump())
params = {
- **run_opts.model_dump(exclude_none=True, by_alias=True),
**job_opts.model_dump(exclude_none=True, by_alias=True),
}
- # run_opts comes from your config; ntasks comes from CLI (or None)
- active_run_opts = merge_model(run_opts, ntasks=ntasks)
-
- # Use this ONE source to update params and to compute mpi_str
- run_params = active_run_opts.as_params()
- params.update(run_params)
- mpi_str = run_params["mpi"] # guaranteed consistent with ntasks override
-
job_kwargs = {
k: v
- for k, v in {"queue": queue, "walltime": walltime, "nodes": nodes, "output_path": log_path.resolve()}.items()
+ for k, v in {
+ "nodes": config_cli.get("nodes"),
+ "ntasks": config_cli.get("ntasks"),
+ "queue": config_cli.get("queue"),
+ "output_path": log_path.resolve(),
+ "tasks": config_cli.get("tasks"),
+ "walltime": config_cli.get("walltime"),
+ }.items()
if v is not None
}
if job_kwargs:
params.update(JobConfig(**job_kwargs).as_params())
+ outline_file = str(Path(outline_file).resolve()) if (outline_file is not None) else "none"
run_toml = {
- "basin": {"basin": "Mouginot/Rignot", "outline": str(outline_file.resolve())},
+ "basin": {"basin": "Mouginot/Rignot", "outline": outline_file},
"output": {
"spatial": str(spatial_file.resolve()),
"state": str(state_file.resolve()),
@@ -286,10 +314,18 @@ def run_kitp(
with open(post_file, "w", encoding="utf-8") as toml_file:
toml.dump(run_toml, toml_file)
- prefix = f"{mpi_str} {cfg.run.executable} "
+ params.update({"run_str": run_str})
+ params.update({"geometry_file": outline_file})
+
+ post_path = output_path / Path("post_processing")
+ post_path.mkdir(parents=True, exist_ok=True)
+
+ post_file = post_path / Path(f"g{resolution}_{name_options}_{start}_{end}.toml")
+ with open(post_file, "w", encoding="utf-8") as toml_file:
+ toml.dump(run_toml, toml_file)
+
postfix = f"pism-kitp-postprocess {post_file}"
rendered_script = "" if debug else template.render(params)
- rendered_script += f"\n\n{prefix}{run_str}\n\n{postfix}"
run_script_path = path / Path("run_scripts")
run_script_path.mkdir(parents=True, exist_ok=True)
@@ -330,7 +366,13 @@ def run_single():
)
parser.add_argument(
"--ntasks",
- help="Overrides ntatsks in config file.",
+ help="Numbers of cores.",
+ type=int,
+ default=None,
+ )
+ parser.add_argument(
+ "--tasks",
+ help="Cores per node.",
type=int,
default=None,
)
@@ -346,18 +388,42 @@ def run_single():
type=str,
default=None,
)
+ parser.add_argument(
+ "--stress-balance",
+ help="Overrides stress balance in config file.",
+ type=str,
+ default=None,
+ )
parser.add_argument(
"--resolution",
help="Override horizontal grid resolution.",
type=str,
default=None,
)
+ parser.add_argument(
+ "--start",
+ help="Override the time.start selection.",
+ type=str,
+ default=None,
+ )
+ parser.add_argument(
+ "--end",
+ help="Override the time.end selection.",
+ type=str,
+ default=None,
+ )
parser.add_argument(
"--debug",
help="Debug or testing mode, do not write template, just the run command.",
action="store_true",
default=False,
)
+ parser.add_argument(
+ "--pism-config-cdl",
+ help="Path to PISM CDL config file for option validation.",
+ type=str,
+ default=None,
+ )
parser.add_argument(
"CONFIG_FILE",
help="CONFIG TOML.",
@@ -379,23 +445,41 @@ def run_single():
queue = options.queue
ntasks = options.ntasks
nodes = options.nodes
+ stress_balance = options.stress_balance
+ tasks = options.tasks
walltime = options.walltime
+ start_cli = options.start
+ end_cli = options.end
+ pism_config_cdl = options.pism_config_cdl
+ config_cli = {
+ "resolution": resolution,
+ "queue": queue,
+ "ntasks": ntasks,
+ "nodes": nodes,
+ "stress_balance": stress_balance,
+ "tasks": tasks,
+ "walltime": walltime,
+ "start": start_cli,
+ "end": end_cli,
+ }
path = Path(path)
path.mkdir(parents=True, exist_ok=True)
- input_path = path / Path("input")
- input_path.mkdir(parents=True, exist_ok=True)
output_path = path / Path("output")
output_path.mkdir(parents=True, exist_ok=True)
cfg = load_config(config_file)
campaign_config = cfg.campaign.as_params()
- bucket = campaign_config["bucket"]
- prefix = campaign_config["prefix"]
+ s3_bucket: str = campaign_config["bucket"] if "bucket" in campaign_config else "pism-cloud-data"
+ s3_prefix: str = campaign_config["prefix"] if "prefix" in campaign_config else "kitp/input"
+ version: str = campaign_config["version"] if "version" in campaign_config else "v2"
+ s3_path = f"""{s3_prefix}/{version}"""
- df = stage(campaign_config, bucket=bucket, prefix=prefix, path=path, force_overwrite=force_overwrite)
- outline_file = df["outline_file"].iloc[0]
+ input_path = path / Path(s3_path)
+ input_path.mkdir(parents=True, exist_ok=True)
+
+ df = stage(campaign_config, s3_bucket, s3_path, path, force_overwrite=force_overwrite)
f = Figlet(font="standard")
banner = f.renderText("pism-terra")
@@ -409,22 +493,26 @@ def run_single():
"input.file": row["boot_file"],
"input.regrid.file": row["regrid_file"],
"energy.bedrock_thermal.file": row["heatflux_file"],
+ "geometry.front_retreat.prescribed.file": row["boot_file"],
"grid.file": row["grid_file"],
+ "atmosphere.elevation_change.file": row["boot_file"],
"atmosphere.given.file": row["climate_file"],
+ "ocean.given.file": row["ocean_file"],
+ "surface.pdd.std_dev.file": row["climate_file"],
+ "surface.debm_simple.std_dev.file": row["climate_file"],
+ "surface.debm_simple.albedo_input.file": row["climate_file"],
}
+ outline_file = row["outline_file"] if "outline_file" in row else None
run_kitp(
config_file,
template_file,
outline_file,
path=path,
- resolution=resolution,
- nodes=nodes,
- ntasks=ntasks,
- queue=queue,
- walltime=walltime,
+ config_cli=config_cli,
debug=debug,
uq=uq,
sample=row["sample"] if "sample" in row else idx,
+ pism_config_cdl=pism_config_cdl,
)
@@ -450,7 +538,19 @@ def run_ensemble():
)
parser.add_argument(
"--ntasks",
- help="Overrides ntatsks in config file.",
+ help="Numbers of cores.",
+ type=int,
+ default=None,
+ )
+ parser.add_argument(
+ "--stress-balance",
+ help="Overrides stress balance in config file.",
+ type=str,
+ default=None,
+ )
+ parser.add_argument(
+ "--tasks",
+ help="Cores per node.",
type=int,
default=None,
)
@@ -472,6 +572,18 @@ def run_ensemble():
type=str,
default=None,
)
+ parser.add_argument(
+ "--start",
+ help="Override the time.start selection.",
+ type=str,
+ default=None,
+ )
+ parser.add_argument(
+ "--end",
+ help="Override the time.end selection.",
+ type=str,
+ default=None,
+ )
parser.add_argument(
"--posterior-file",
help="CSV file posterior parameter distributions to sample from. Default=None.",
@@ -484,6 +596,12 @@ def run_ensemble():
action="store_true",
default=False,
)
+ parser.add_argument(
+ "--pism-config-cdl",
+ help="Path to PISM CDL config file for option validation.",
+ type=str,
+ default=None,
+ )
parser.add_argument(
"--force-overwrite",
help="Force downloading all files.",
@@ -518,7 +636,23 @@ def run_ensemble():
queue = options.queue
ntasks = options.ntasks
nodes = options.nodes
+ stress_balance = options.stress_balance
+ tasks = options.tasks
walltime = options.walltime
+ start_cli = options.start
+ end_cli = options.end
+ pism_config_cdl = options.pism_config_cdl
+ config_cli = {
+ "resolution": resolution,
+ "queue": queue,
+ "ntasks": ntasks,
+ "nodes": nodes,
+ "stress_balance": stress_balance,
+ "tasks": tasks,
+ "walltime": walltime,
+ "start": start_cli,
+ "end": end_cli,
+ }
path = Path(path)
path.mkdir(parents=True, exist_ok=True)
@@ -530,20 +664,11 @@ def run_ensemble():
cfg = load_config(config_file)
campaign_config = cfg.campaign.as_params()
- bucket = campaign_config["bucket"]
- prefix = campaign_config["prefix"]
-
- df = stage(campaign_config, bucket=bucket, prefix=prefix, path=path, force_overwrite=force_overwrite)
- outline_file = df["outline_file"].iloc[0]
-
- default = {
- "input.file": df["boot_file"].iloc[0],
- "input.regrid.file": df["regrid_file"].iloc[0],
- "grid.file": df["grid_file"].iloc[0],
- "atmosphere.given.file": df["climate_file"].iloc[0],
- "atmosphere.elevation_change.file": df["climate_file"].iloc[0],
- "energy.bedrock_thermal.file": df["heatflux_file"].iloc[0],
- }
+ s3_bucket: str = campaign_config.get("bucket", "pism-cloud-data")
+ s3_prefix: str = campaign_config.get("prefix", "kitp/input")
+ version: str = campaign_config.get("version", "v2")
+ s3_path = f"""{s3_prefix}/{version}"""
+ df = stage(campaign_config, s3_bucket, s3_path, path, force_overwrite=force_overwrite)
seed = 42
rng = np.random.default_rng(seed=seed)
@@ -551,7 +676,7 @@ def run_ensemble():
n_samples = uq.samples
mapping = uq.mapping
- uq_df = create_samples(uq.to_flat(), n_samples=n_samples, seed=seed)
+ uq_df = generate_samples(uq.to_flat(), n_samples=n_samples, method=uq.method, seed=seed)
if posterior_file is not None:
posterior_df = pd.read_csv(posterior_file).drop(columns=["Unnamed: 0", "exp_id"], errors="ignore")
choice_indices = rng.choice(range(len(posterior_df)), n_samples)
@@ -567,31 +692,47 @@ def run_ensemble():
f = Figlet(font="standard")
banner = f.renderText("pism-terra")
- print("=" * 80)
+ print("=" * 120)
print(banner)
- print("=" * 80)
+ print("=" * 120)
print("Generate Ensemble Runs for Greenland")
- print("-" * 80)
+ print("-" * 120)
+
if uq.mapping:
uq_df = apply_choice_mapping(uq_df, df, uq.mapping)
- for df_idx, df_row in df.iterrows():
- df_sample = df_row["sample"] if "sample" in df_row else df_idx
- for idx, row in uq_df.iterrows():
- sample = (df_sample + "_uq_" + str(int((row["sample"])))) if "sample" in row else (df_idx + "_" + idx)
- run_kitp(
- config_file,
- template_file,
- outline_file,
- path=path,
- resolution=resolution,
- nodes=nodes,
- ntasks=ntasks,
- queue=queue,
- walltime=walltime,
- debug=debug,
- uq=default,
- sample=sample,
- )
+
+ merged_df = df.merge(uq_df, how="cross", suffixes=("_df", "_uq"))
+ merged_df["sample"] = merged_df["sample_df"].astype(str) + "_uq_" + merged_df["sample_uq"].astype(int).astype(str)
+ merged_df = merged_df.drop(columns=["sample_df", "sample_uq"])
+
+ for _, row in merged_df.iterrows():
+ row_uq = {
+ "input.file": row["boot_file"],
+ "input.regrid.file": row["regrid_file"],
+ "energy.bedrock_thermal.file": row["heatflux_file"],
+ "geometry.front_retreat.prescribed.file": row["boot_file"],
+ "grid.file": row["grid_file"],
+ "atmosphere.elevation_change.file": row["boot_file"],
+ "atmosphere.given.file": row["climate_file"],
+ "ocean.given.file": row["ocean_file"],
+ "surface.pdd.std_dev.file": row["climate_file"],
+ "surface.debm_simple.std_dev.file": row["climate_file"],
+ "surface.debm_simple.albedo_input.file": row["climate_file"],
+ }
+ row_uq.update(row.drop(labels=list(df.columns) + ["sample"]).to_dict())
+ outline_file = row["outline_file"] if "outline_file" in row else None
+ sample = row["sample"]
+ run_kitp(
+ config_file,
+ template_file,
+ outline_file,
+ path=path,
+ config_cli=config_cli,
+ debug=debug,
+ uq=row_uq,
+ sample=sample,
+ pism_config_cdl=pism_config_cdl,
+ )
if __name__ == "__main__":
diff --git a/pism_terra/kitp/stage.py b/pism_terra/kitp/stage.py
index cc5b6cf..71bdd5f 100644
--- a/pism_terra/kitp/stage.py
+++ b/pism_terra/kitp/stage.py
@@ -24,7 +24,7 @@
import time
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from collections.abc import Mapping
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path
from typing import Callable
@@ -35,6 +35,7 @@
import xarray as xr
from pyfiglet import Figlet
from shapely.geometry import Polygon
+from tqdm.auto import tqdm
from pism_terra.aws import local_to_s3, s3_to_local
from pism_terra.config import load_config
@@ -45,17 +46,18 @@
def stage(
config: dict,
- path: str | Path = "input_files",
- bucket: str = "pism-cloud-data",
- prefix: str = "kitp/input",
+ bucket: str,
+ prefix: str,
+ output_path: str | Path,
force_overwrite: bool = False,
) -> pd.DataFrame:
"""
Stage KITP Greenland inputs and return a file index.
Syncs pre-built input data from S3, validates each file, and returns
- a single-row DataFrame with absolute paths to all staged artifacts
- (boot, grid, heatflux, regrid, retreat, climate, and ocean files).
+ a DataFrame with absolute paths to all staged artifacts (boot, grid,
+ heatflux, regrid, ocean, outline, and climate files), one row per
+ GCM/forcing combination plus a baseline-climatology row.
Parameters
----------
@@ -69,16 +71,22 @@ def stage(
Path to the heatflux NetCDF file relative to the input directory.
- ``"regrid_file"`` : str
Path to the regrid NetCDF file relative to the input directory.
- - ``"gcm"`` : str
- GCM model name.
+ - ``"ocean_file"`` : str
+ Path to the ocean-forcing NetCDF file relative to the input directory.
+ - ``"outline_file"`` : str
+ Path to the basin outline file relative to the input directory.
+ - ``"gcms"`` : dict[str, list[list[str]]]
+ Mapping of GCM names to their forcing pairs ``[[future, present], ...]``.
+ - ``"climatology"`` : str
+ Baseline climatology stem used to build climate filenames.
- ``"version"`` : str
Dataset version.
- path : str or pathlib.Path, default ``"input_files"``
- Output directory. Created if missing. All staged artifacts are written here.
- bucket : str, default ``"pism-cloud-data"``
+ bucket : str
AWS S3 bucket name to sync KITP input data from.
- prefix : str, default ``"kitp_greenland_input"``
+ prefix : str
S3 key prefix (folder path within the bucket).
+ output_path : str or pathlib.Path`
+ Output directory. Created if missing. All staged artifacts are written here.
force_overwrite : bool, default ``False``
If ``True``, downstream helpers may regenerate intermediate/final artifacts
even if cache files exist.
@@ -86,10 +94,10 @@ def stage(
Returns
-------
pandas.DataFrame
- Single-row DataFrame with absolute-path columns including
- ``boot_file``, ``grid_file``, ``heatflux_file``, ``regrid_file``,
- ``retreat_file``, ``climate_file``, ``ocean_file``,
- ``surface_input_file``, and ``frontal_melt_file``.
+ DataFrame with one row per GCM/forcing pair (plus a baseline-only
+ row), with absolute-path columns ``boot_file``, ``grid_file``,
+ ``heatflux_file``, ``ocean_file``, ``outline_file``, ``regrid_file``,
+ and ``climate_file``, plus a ``sample`` identifier column.
"""
f = Figlet(font="standard")
@@ -102,62 +110,99 @@ def stage(
print("")
# Outputs dir
- path = Path(path)
- path.mkdir(parents=True, exist_ok=True)
+ output_path = Path(output_path)
+ output_path.mkdir(parents=True, exist_ok=True)
+
+ input_path = output_path / Path(prefix)
- input_path = path / Path("input")
if force_overwrite:
input_path.unlink(missing_ok=True)
input_path.mkdir(parents=True, exist_ok=True)
-
s3_to_local(bucket, prefix=prefix, dest=input_path)
grid_file = input_path / Path(config["grid_file"])
+ # Grid file gets a heavier check (full load) so leave it sequential.
check_xr_fully(grid_file)
boot_file = input_path / Path(config["boot_file"])
- check_xr_lazy(boot_file)
-
heatflux_file = input_path / Path(config["heatflux_file"])
- check_xr_lazy(heatflux_file)
-
regrid_file = input_path / Path(config["regrid_file"])
- check_xr_lazy(regrid_file)
-
+ ocean_file = input_path / Path(config["ocean_file"])
outline_file = input_path / Path(config["outline_file"])
+ # Validate the lazy-check inputs concurrently; only invalid files print.
+ input_lazy_files = [boot_file, heatflux_file, regrid_file, ocean_file]
+ # Processes (not threads): HDF5 isn't reliably thread-safe across all
+ # builds (Chinook segfaults), so each worker gets its own interpreter
+ # and HDF5 state.
+ with ProcessPoolExecutor(max_workers=8) as executor:
+ future_to_path = {executor.submit(check_xr_lazy, p, verbose=False): p for p in input_lazy_files}
+ for future in tqdm(
+ as_completed(future_to_path),
+ total=len(future_to_path),
+ desc="Checking input files",
+ unit="file",
+ ):
+ p = future_to_path[future]
+ if not future.result():
+ print(f"{p.resolve()} is not valid ✗")
+
# Build file index (one row per climate file)
files_dict: dict[str, str | Path] = {
"boot_file": boot_file.resolve(),
"grid_file": grid_file.resolve(),
"heatflux_file": heatflux_file.resolve(),
- "regrid_file": regrid_file.resolve(),
+ "ocean_file": ocean_file.resolve(),
"outline_file": outline_file.resolve(),
+ "regrid_file": regrid_file.resolve(),
}
- for key in ("gcms", "present_day_forcings", "future_forcings"):
- if isinstance(config[key], str):
- config[key] = [config[key]]
-
gcms = config["gcms"]
+ climatology = config["climatology"]
version = config["version"]
- present_day_forcings = config["present_day_forcings"]
- future_forcings = config["future_forcings"]
- tasks = [(gcm, pd_forcing, ff) for gcm in gcms for pd_forcing in present_day_forcings for ff in future_forcings]
+ # Build tasks from per-GCM forcing pairs [[future, present], ...]
+ tasks = [(gcm, pair[1], pair[0]) for gcm, pairs in gcms.items() for pair in pairs]
dfs: list[pd.DataFrame] = []
- climate_file = input_path / Path(f"HIRHAM5-ERA5_YMM_1990_2019_{version}.nc")
- check_xr_lazy(climate_file)
- files_dict["climate_file"] = climate_file.resolve()
- files_dict["sample"] = "HIRHAM5-ERA5_YMM_1990_2019"
- dfs.append(pd.DataFrame.from_dict([files_dict]))
- for task in tasks:
- gcm, pd_forcing, ff = task
- climate_file = input_path / Path(f"HIRHAM5-ERA5_YMM_1990_2019_{gcm}_anomalies_{ff}_{pd_forcing}_{version}.nc")
- check_xr_lazy(climate_file)
- files_dict["climate_file"] = climate_file.resolve()
- files_dict["sample"] = f"{gcm}_{ff}_{pd_forcing}"
- dfs.append(pd.DataFrame.from_dict([files_dict]))
+
+ # Enumerate every climate file we'll need (climatology baseline + one per
+ # GCM/forcing combo) so the validation can run in parallel below.
+ climate_specs: list[tuple[str, Path]] = [
+ (climatology, input_path / Path(f"{climatology}_{version}.nc")),
+ ]
+ for gcm, pd_forcing, ff in tasks:
+ climate_specs.append(
+ (
+ f"gcm_{gcm}_exp_{ff}_{pd_forcing}",
+ input_path / Path(f"{climatology}_{gcm}_anomalies_{ff}_{pd_forcing}_{version}.nc"),
+ )
+ )
+
+ # Validate climate files concurrently. ``check_xr_lazy`` is mostly I/O
+ # (open netCDF, sample a window); a worker pool overlaps the latency.
+ # Suppress its per-file ✓ chatter and only surface invalid files here.
+ invalid_climate_files: list[Path] = []
+ # Processes (not threads): HDF5 isn't reliably thread-safe across all
+ # builds (Chinook segfaults), so each worker gets its own interpreter
+ # and HDF5 state.
+ with ProcessPoolExecutor(max_workers=8) as executor:
+ future_to_path = {executor.submit(check_xr_lazy, p, verbose=False): p for _, p in climate_specs}
+ for future in tqdm(
+ as_completed(future_to_path),
+ total=len(future_to_path),
+ desc="Checking climate files",
+ unit="file",
+ ):
+ p = future_to_path[future]
+ if not future.result():
+ invalid_climate_files.append(p)
+ print(f"{p.resolve()} is not valid ✗")
+
+ for sample, climate_file in climate_specs:
+ row = dict(files_dict)
+ row["climate_file"] = climate_file.resolve()
+ row["sample"] = sample
+ dfs.append(pd.DataFrame.from_dict([row]))
df = pd.concat(dfs).reset_index(drop=True)
return df
@@ -196,21 +241,25 @@ def main():
)
options, unknown = parser.parse_known_args()
- path = options.output_path
config_file = options.CONFIG_FILE[0]
force_overwrite = options.force_overwrite
+ output_path = options.output_path
+ output_path.mkdir(parents=True, exist_ok=True)
cfg = load_config(config_file)
config = cfg.campaign.as_params()
- path.mkdir(parents=True, exist_ok=True)
+ s3_bucket: str = config.pop("bucket", "pism-cloud-data")
+ s3_prefix: str = config.pop("prefix", "kitp/input")
+ version: str = config.pop("version", "v2")
+ s3_path = f"""{s3_prefix}/{version}"""
- is_df = stage(config, path=path, force_overwrite=force_overwrite)
- is_df.to_csv(path / Path("input") / Path("ismip7_greenland_files.csv"))
+ is_df = stage(config, s3_bucket, s3_path, output_path, force_overwrite=force_overwrite)
+ is_df.to_csv(output_path / Path(s3_path) / Path("ismip7_greenland_files.csv"))
if options.bucket:
prefix = f"{options.bucket_prefix}/kitp_greenland" if options.bucket_prefix else "kitp_greenland"
- local_to_s3(path, bucket=options.bucket, prefix=prefix)
+ local_to_s3(output_path, bucket=options.bucket, prefix=prefix)
if __name__ == "__main__":
diff --git a/pism_terra/likelihood.py b/pism_terra/likelihood.py
index f59ea64..2a1b690 100644
--- a/pism_terra/likelihood.py
+++ b/pism_terra/likelihood.py
@@ -178,7 +178,7 @@ def log_normal_xr(
da = da.where(da != 0, np.nan)
da.name = "log_likelihood"
da.attrs.update({"units": "1", "long_name": "negative log likelihood"})
- return da.sum(dim=sum_dims)
+ return da.mean(dim=sum_dims)
def log_pseudo_huber(
@@ -249,4 +249,4 @@ def log_pseudo_huber_xr(
da = da.where(da != 0, np.nan)
da.name = "log_likelihood"
da.attrs.update({"units": "1", "long_name": "negative log likelihood"})
- return da.sum(dim=sum_dims)
+ return da.mean(dim=sum_dims)
diff --git a/pism_terra/log.py b/pism_terra/log.py
new file mode 100644
index 0000000..b073adb
--- /dev/null
+++ b/pism_terra/log.py
@@ -0,0 +1,63 @@
+# Copyright (C) 2025, 2026 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+"""
+Centralized logging configuration.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+
+def setup_logging(log_file: str | Path) -> None:
+ """
+ Configure logging for pism-terra CLI entry points.
+
+ Sets up two handlers:
+
+ - **Console** (root): WARNING level so per-step INFO chatter does not
+ interleave with tqdm progress bars on the terminal.
+ - **File**: INFO level with full
+ ``%(asctime)s - %(name)s - %(levelname)s - %(message)s`` format for
+ detailed log files.
+
+ The ``pism_terra`` logger is set to INFO so all package-level INFO records
+ are captured by the file handler (and bubble up to console only at WARNING+).
+
+ Parameters
+ ----------
+ log_file : str or Path
+ Path to the log file. Parent directories must already exist.
+ """
+ file_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+ logging.basicConfig(level=logging.WARNING, format=file_format)
+ for handler in logging.root.handlers:
+ handler.setLevel(logging.WARNING)
+
+ file_handler = logging.FileHandler(log_file)
+ file_handler.setLevel(logging.INFO)
+ file_handler.setFormatter(logging.Formatter(file_format))
+
+ pkg_logger = logging.getLogger("pism_terra")
+ pkg_logger.setLevel(logging.INFO)
+ pkg_logger.addHandler(file_handler)
+
+ # Quiet noisy third-party INFO chatter (CDS API, AWS, etc.) on the console.
+ for name in ("cdsapi", "datapi", "multiurl", "ecmwf", "botocore", "s3transfer", "boto3"):
+ logging.getLogger(name).setLevel(logging.WARNING)
diff --git a/pism_terra/plotting.py b/pism_terra/plotting.py
new file mode 100644
index 0000000..6c753fc
--- /dev/null
+++ b/pism_terra/plotting.py
@@ -0,0 +1,47 @@
+# Copyright (C) 2025 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+"""
+Plotting methods.
+"""
+
+import numpy as np
+
+
+def blend_multiply(rgb: np.ndarray, intensity: np.ndarray) -> np.ndarray:
+ """
+ Combine an RGB image with an intensity map using "overlay" blending.
+
+ This function combines an RGB image with an intensity map using "overlay" blending. The RGB image
+ and the intensity map are combined by multiplying the RGB values by the intensity values. The resulting
+ image is then scaled to have values between 0 and 1.
+
+ Parameters
+ ----------
+ rgb : np.ndarray
+ An (M, N, 3) RGB array of floats ranging from 0 to 1. This represents the color image.
+ intensity : np.ndarray
+ An (M, N, 1) array of floats ranging from 0 to 1. This represents the grayscale image.
+
+ Returns
+ -------
+ np.ndarray
+ An (M, N, 3) RGB array representing the combined images. The values in the array range from 0 to 1.
+ """
+
+ alpha = rgb[..., -1, np.newaxis]
+ img_scaled = np.clip(rgb[..., :3] * intensity, 0.0, 1.0)
+ return img_scaled * alpha + intensity * (1.0 - alpha)
diff --git a/pism_terra/prepare_select.py b/pism_terra/prepare_select.py
new file mode 100644
index 0000000..1ea47af
--- /dev/null
+++ b/pism_terra/prepare_select.py
@@ -0,0 +1,82 @@
+# Copyright (C) 2026 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+"""
+Shared helpers for the ``--include`` dataset selector used by prepare CLIs.
+"""
+
+from __future__ import annotations
+
+import logging
+from argparse import ArgumentParser
+from collections.abc import Sequence
+
+logger = logging.getLogger(__name__)
+
+
+def add_include_argument(parser: ArgumentParser, available: Sequence[str]) -> None:
+ """
+ Add a ``--include`` option that selects which datasets to process.
+
+ Parameters
+ ----------
+ parser : argparse.ArgumentParser
+ Parser to extend.
+ available : sequence of str
+ Dataset names this command can process, listed in the help text.
+ """
+ parser.add_argument(
+ "--include",
+ default=None,
+ metavar="DATASET[,DATASET...]",
+ help=(
+ "Comma-separated list of datasets to process; if omitted, all are "
+ f"processed. Available: {', '.join(available)}."
+ ),
+ )
+
+
+def select_datasets(include: str | None, available: Sequence[str]) -> list[str]:
+ """
+ Resolve ``--include`` to an ordered subset of ``available`` datasets.
+
+ Parameters
+ ----------
+ include : str or None
+ Raw ``--include`` value (comma-separated), or ``None`` for "all".
+ available : sequence of str
+ Canonical dataset names; the returned list preserves this order.
+
+ Returns
+ -------
+ list of str
+ Selected dataset names.
+
+ Raises
+ ------
+ SystemExit
+ If ``include`` names a dataset not in ``available``.
+ """
+ if not include:
+ return list(available)
+ requested = {s.strip() for s in include.split(",") if s.strip()}
+ unknown = sorted(requested - set(available))
+ if unknown:
+ raise SystemExit(f"Unknown dataset(s) in --include: {', '.join(unknown)}. Available: {', '.join(available)}.")
+ selected = [d for d in available if d in requested]
+ logger.info("Processing datasets: %s", ", ".join(selected))
+ return selected
diff --git a/pism_terra/processing.py b/pism_terra/processing.py
index db7cca2..c5a9faa 100644
--- a/pism_terra/processing.py
+++ b/pism_terra/processing.py
@@ -23,6 +23,7 @@
Processing Functions.
"""
+import json
import re
from collections import OrderedDict
from collections.abc import Hashable, Mapping
@@ -32,12 +33,16 @@
import xarray as xr
-def preprocess_nc(
- ds: xr.Dataset,
- regexp: str = "id_(.+?)_",
- dim: str = "exp_id",
+def preprocess_netcdf(
+ ds,
+ exp_regexp: str = "id_(.+?)_",
+ uq_regexp: str | None = r"(RGI2000-v7\.0-C-[^/\s]+)",
+ exp_dim: str = "exp_id",
+ uq_dim: str | None = "uq_id",
+ gcm_dim: str | None = "gcm_id",
drop_vars: list[str] | None = None,
- drop_dims: list[str] | None = None,
+ drop_dims: list[str] = ["nv4"],
+ process_config: bool = True,
) -> xr.Dataset:
"""
Add experiment identifier to the dataset.
@@ -50,48 +55,96 @@ def preprocess_nc(
----------
ds : xarray.Dataset
The input dataset to be processed.
- regexp : str, optional
- The regular expression pattern to extract the experiment identifier from the filename,
- by default "id_(.+?)_".
- dim : str, optional
- The name of the new dimension to be added to the dataset, by default "exp_id".
- drop_vars : list of str or None, optional
+ exp_regexp : str, optional
+ The regular expression pattern to extract the experiment identifier from the filename, by default "id_(.+?)_".
+ uq_regexp : str or None, optional
+ The regular expression pattern to extract the UQ identifier from the filename, by default ``r"(RGI2000-v7\\.0-C-[^/\\s]+)"``.
+ If None, no UQ dimension is added.
+ exp_dim : str, optional
+ The name of the new experiment dimension to be added to the dataset, by default "exp_id".
+ uq_dim : str or None, optional
+ The name of the new UQ dimension to be added to the dataset, by default "uq_id".
+ If None, no UQ dimension is added.
+ gcm_dim : str or None, optional
+ The name of the GCM dimension to be added to the dataset, by default "gcm_id".
+ If None, no GCM dimension is added. The GCM name is extracted from the filename
+ by matching the pattern ``id__``.
+ drop_vars : list[str]| None, optional
A list of variable names to be dropped from the dataset, by default None.
- drop_dims : list of str or None, optional
- A list of dimension names to be dropped from the dataset, by default None.
+ drop_dims : list[str], optional
+ A list of dimension names to be dropped from the dataset, by default ["nv4"].
+ process_config : bool, optional
+ If True, extract and store pism_config as a JSON-encoded DataArray. If False, simply
+ drop the pism_config variable and axis without re-adding it. By default True.
Returns
-------
xarray.Dataset
- The processed dataset with the experiment identifier added as a new dimension,
- and specified variables and dimensions dropped.
+ The processed dataset with the experiment identifier added as a new dimension, and specified variables and dimensions dropped.
Raises
------
AssertionError
If the regular expression does not match any part of the filename.
-
- Notes
- -----
- If `drop_dims` is not provided, it defaults to `["nv4"]`.
"""
- if drop_dims is None: # Initialize drop_dims if not provided
- drop_dims = ["nv4"]
-
- m_id_re = re.search(regexp, ds.encoding["source"])
- ds = ds.expand_dims(dim)
- assert m_id_re is not None
- m_id: str | int
- try:
- m_id = int(m_id_re.group(1))
- except ValueError: # Catch specific exception
- m_id = str(m_id_re.group(1))
- ds[dim] = [m_id]
+
+ m_exp_id_re = re.search(exp_regexp, ds.encoding["source"])
+ assert m_exp_id_re is not None
+ m_exp_id = m_exp_id_re.group(1)
+
+ if process_config:
+ p_config = ds["pism_config"]
+
+ ds = ds.drop_vars(["pism_config"], errors="ignore").drop_dims(["pism_config_axis"], errors="ignore")
+
+ expand_dims = []
+ expand_coords = {}
+
+ if gcm_dim is not None:
+ gcm_regexp = r"_gcm_(.+?)_exp_"
+ m_gcm_re = re.search(gcm_regexp, ds.encoding["source"])
+ if m_gcm_re is not None:
+ m_gcm_id = m_gcm_re.group(1)
+ expand_dims.append(gcm_dim)
+ expand_coords[gcm_dim] = [m_gcm_id]
+
+ if uq_regexp is not None and uq_dim is not None and hasattr(ds, "command"):
+ m_uq_id_re = re.search(uq_regexp, ds.command)
+ assert m_uq_id_re is not None
+ m_uq_id = m_uq_id_re.group(1)
+ expand_dims.append(uq_dim)
+ expand_coords[uq_dim] = [m_uq_id]
+
+ expand_dims.append(exp_dim)
+ expand_coords[exp_dim] = [m_exp_id]
+ ds = ds.expand_dims(expand_coords)
+
+ if process_config:
+
+ # List of suffixes to exclude
+ suffixes_to_exclude = ["_doc", "_type", "_units", "_option", "_choices"]
+
+ # Filter the dictionary and encode as a single JSON string per (uq_id, exp_id)
+ config = {
+ k: v for k, v in p_config.attrs.items() if not any(k.endswith(suffix) for suffix in suffixes_to_exclude)
+ }
+ if "geometry.front_retreat.prescribed.file" not in config.keys():
+ config["geometry.front_retreat.prescribed.file"] = "false"
+
+ config_json = json.dumps(OrderedDict(sorted(config.items())))
+ shape = [1] * len(expand_dims)
+ pism_config = xr.DataArray(
+ np.array(config_json, dtype=object).reshape(shape),
+ dims=expand_dims,
+ coords=expand_coords,
+ name="pism_config",
+ )
+ ds = ds.assign_coords(pism_config=pism_config)
return ds.drop_vars(drop_vars, errors="ignore").drop_dims(drop_dims, errors="ignore")
-def preprocess_config(
+def preprocess_config_rgi(
ds,
exp_regexp: str = "id_(.+?)_",
rgi_regexp: str = r"(RGI2000-v7\.0-C-[^/\s]+)",
@@ -144,34 +197,26 @@ def preprocess_config(
m_exp_id = m_exp_id_re.group(1)
p_config = ds["pism_config"]
+ ds = ds.drop_vars(["pism_config"], errors="ignore").drop_dims(["pism_config_axis"], errors="ignore")
ds = ds.expand_dims({rgi_dim: [m_rgi_id], exp_dim: [m_exp_id]})
# List of suffixes to exclude
suffixes_to_exclude = ["_doc", "_type", "_units", "_option", "_choices"]
- # Filter the dictionary
+ # Filter the dictionary and encode as a single JSON string per (rgi_id, exp_id)
config = {k: v for k, v in p_config.attrs.items() if not any(k.endswith(suffix) for suffix in suffixes_to_exclude)}
if "geometry.front_retreat.prescribed.file" not in config.keys():
config["geometry.front_retreat.prescribed.file"] = "false"
- config_sorted = OrderedDict(sorted(config.items()))
-
- pc_keys = np.array(list(config_sorted.keys()))
- pc_vals = np.array(list(config_sorted.values()))
-
+ config_json = json.dumps(OrderedDict(sorted(config.items())))
pism_config = xr.DataArray(
- pc_vals.reshape(-1),
- dims=["pism_config_axis"],
- coords={"pism_config_axis": pc_keys},
+ np.array([[config_json]], dtype=object),
+ dims=[rgi_dim, exp_dim],
+ coords={rgi_dim: [m_rgi_id], exp_dim: [m_exp_id]},
name="pism_config",
)
+ ds = ds.assign_coords(pism_config=pism_config)
- ds = xr.merge(
- [
- ds.drop_vars(["pism_config"], errors="ignore").drop_dims(["pism_config_axis"], errors="ignore"),
- pism_config,
- ]
- )
return ds.drop_vars(drop_vars, errors="ignore").drop_dims(drop_dims, errors="ignore")
diff --git a/pism_terra/profiles.py b/pism_terra/profiles.py
index f844aa7..a2ff14f 100644
--- a/pism_terra/profiles.py
+++ b/pism_terra/profiles.py
@@ -135,9 +135,9 @@ def calculate_stats(
Parameters
----------
obs_var : str, optional
- The observed data variable name in the xarray Dataset, by default "v".
+ The observed data variable name in the xarray Dataset, by default "obs_v_normal".
sim_var : str, optional
- The simulated data variable name in the xarray Dataset, by default "velsurf_mag".
+ The simulated data variable name in the xarray Dataset, by default "sim_v_normal".
dim : str, optional
The dimension along which to calculate the statistics, by default "profile_axis".
stats : List[str], optional
diff --git a/pism_terra/raster.py b/pism_terra/raster.py
index 80fd4dd..db41343 100644
--- a/pism_terra/raster.py
+++ b/pism_terra/raster.py
@@ -22,8 +22,8 @@
Provide raster functions.
"""
+import math
from pathlib import Path
-from tempfile import NamedTemporaryFile
import geopandas as gpd
import numpy as np
@@ -32,7 +32,7 @@
import rioxarray as rxr
import xarray as xr
from geocube.api.core import make_geocube
-from rasterio.warp import Resampling, calculate_default_transform, reproject
+from pyproj import CRS, Transformer
from shapely.geometry import box
from pism_terra.workflow import check_xr_lazy
@@ -109,7 +109,7 @@ def create_ds(
encoding.update(encoding_time)
encoding.update({var: {"_FillValue": None} for var in list(ds.data_vars) + list(ds.coords)})
- ds.to_netcdf(output_file, encoding=encoding)
+ ds.to_netcdf(output_file, encoding=encoding, engine="h5netcdf")
return output_file
@@ -302,50 +302,173 @@ def raster_overlaps_glacier(
return glacier_box.intersects(raster_box)
-def reproject_file(src_file: str | Path, dst_crs: str | dict, resolution: float) -> str:
+def local_scale_factor(
+ crs,
+ x,
+ y,
+ eps_deg: float = 1.0e-6,
+):
"""
- Reproject a raster file to a new coordinate reference system and resolution.
-
- This function opens a source raster file, reprojects its contents to a specified
- destination CRS and resolution using average resampling, and writes the result
- to a temporary GeoTIFF file. The path to this reprojected file is returned.
+ Local linear scale factor of a projected CRS at a point.
+
+ Returns the dimensionless number ``k`` such that **1 metre of true
+ ground displacement at ``(x, y)`` corresponds to ``k`` metres measured
+ in ``crs`` map units**. ``k = 1`` for an isometric projection (e.g.
+ polar stereographic near its true-scale latitude); ``k > 1`` where the
+ projection inflates distances (e.g. Web Mercator at high latitudes;
+ ``k ≈ 1/cos(lat)`` there); ``k < 1`` where it shrinks them.
+
+ The factor is computed as ``sqrt(|det J|)``, where ``J`` is the
+ Jacobian of the *true-to-map* coordinate transform at the point:
+
+ .. math::
+
+ J = \\begin{pmatrix}
+ \\partial x_\\mathrm{crs} / \\partial x_\\mathrm{true} &
+ \\partial x_\\mathrm{crs} / \\partial y_\\mathrm{true} \\\\
+ \\partial y_\\mathrm{crs} / \\partial x_\\mathrm{true} &
+ \\partial y_\\mathrm{crs} / \\partial y_\\mathrm{true}
+ \\end{pmatrix}
+
+ The Jacobian is estimated by finite-differencing the
+ geographic-to-map transform a small distance from the point. The
+ geographic step (in degrees) is converted to a true-metres step using
+ the **prime-vertical radius of curvature** ``N(lat)`` along longitude
+ and the **meridional radius of curvature** ``M(lat)`` along latitude,
+ both evaluated on the source CRS's own ellipsoid (no spherical-Earth
+ approximation):
+
+ .. math::
+
+ e^2 = 2f - f^2, \\quad
+ N(\\varphi) = \\frac{a}{\\sqrt{1 - e^2 \\sin^2 \\varphi}}, \\quad
+ M(\\varphi) = \\frac{a (1 - e^2)}{(1 - e^2 \\sin^2 \\varphi)^{3/2}}
+
+ where ``a`` is the ellipsoid's semi-major axis and ``f`` its
+ flattening. One degree of latitude corresponds to ``M·π/180`` metres;
+ one degree of longitude at latitude ``φ`` corresponds to
+ ``N·cos(φ)·π/180`` metres. Using these to normalise the Jacobian gives
+ ``det J`` in units of ``m_crs² / m_true²``; its square root is the
+ scalar scale factor.
+
+ For **conformal** CRSs (Mercator, polar stereographic, UTM,
+ Lambert conformal conic, …) the projection is locally isotropic, so
+ ``k_x = k_y = k`` — the single returned number fully describes the
+ distortion. For **non-conformal** CRSs (Albers equal-area, …)
+ ``k_x ≠ k_y``; ``sqrt(|det J|)`` is the area-equivalent (geometric
+ mean) scale and is good enough for vector-magnitude corrections.
Parameters
----------
- src_file : str or Path
- Path to the source raster file.
- dst_crs : str or dict
- Destination coordinate reference system (e.g., "EPSG:32633" or a CRS dict).
- resolution : float
- Target resolution for the output raster in units of the destination CRS.
+ crs : str or pyproj.CRS
+ Source CRS for the velocity (or other vector) field. Anything
+ :class:`pyproj.CRS` accepts: ``"EPSG:3857"``, WKT, PROJ4, etc.
+ x, y : float or numpy.ndarray
+ Position(s) in ``crs`` units (typically metres) where the scale
+ factor is evaluated. Scalars or broadcastable arrays.
+ eps_deg : float, default ``1.0e-6``
+ Geographic step (degrees) used for the numerical Jacobian. The
+ default keeps the truncation error below 1 part in 10⁵ for the
+ projections used in glaciology while staying well above floating
+ -point precision.
Returns
-------
- str
- Path to the temporary reprojected raster file (GeoTIFF).
+ float or numpy.ndarray
+ Local scale factor ``k`` (dimensionless). Same shape as ``x``/``y``.
Notes
-----
- - The output file is written to a temporary location and is not automatically deleted.
- It is the caller's responsibility to clean it up.
- - The reprojected data is resampled using `Resampling.average`.
+ **How to use this to correct a finite-difference velocity transform.**
+
+ Vector products like ITS_LIVE store velocity components as the
+ *true ground motion* in m/yr, projected onto the raster CRS's
+ coordinate axes. The advect-and-roundtrip pattern used by
+ :func:`pism_terra.glacier.observations.glacier_velocities_from_grid`,
+ however, implicitly assumes the components describe the *rate of
+ change of the map coordinate* (i.e. ``v = dx_crs/dt``). The two
+ conventions agree when the source CRS has unit scale factor at the
+ point of interest (polar stereographic over Greenland or Antarctica
+ near 70°S/70°N, UTM near its central meridian), but diverge wherever
+ ``k != 1`` — most visibly with Web Mercator (EPSG:3857), where
+ ``k = 1/cos(φ) ≈ 2`` at 60° latitude.
+
+ To make the FD round-trip recover the right magnitude regardless of
+ the source CRS, pre-multiply the source-side velocity by the local
+ scale factor before advecting:
+
+ .. code-block:: python
+
+ from pism_terra.raster import local_scale_factor
+
+ k = local_scale_factor(src_crs, X_, Y_) # X_, Y_ in src_crs metres
+ vx_pts *= k
+ vy_pts *= k
+ # ... continue with the existing FD round-trip; the result is now
+ # in m/yr aligned with the destination CRS axes, independent of
+ # the source CRS's local distortion.
+
+ Equivalently, divide by ``k`` *after* the round-trip. For an isometric
+ source (``k ≈ 1``) the correction is a no-op, so it's safe to apply
+ unconditionally as a defensive measure.
+
+ Examples
+ --------
+ Mercator (EPSG:3857) inflates distances by ``1/cos(φ)``. At about 60° N
+ the local scale factor is therefore close to 2:
+
+ >>> from pism_terra.raster import local_scale_factor
+ >>> round(local_scale_factor("EPSG:3857", 0.0, 8362900.0), 2)
+ 1.99
+
+ Polar stereographic NSIDC (EPSG:3413) has true scale at 70° N, so the
+ factor is close to 1 over most of the Greenland Ice Sheet:
+
+ >>> round(local_scale_factor("EPSG:3413", 0.0, -1_000_000.0), 2)
+ 0.98
+
+ Vectorised call — pass arrays of points and get an array back:
+
+ >>> import numpy as np
+ >>> ks = local_scale_factor("EPSG:3857", np.zeros(3), np.linspace(0, 8e6, 3))
+ >>> [round(float(k), 2) for k in ks]
+ [1.0, 1.2, 1.89]
"""
- with rasterio.open(src_file) as src:
- transform, width, height = calculate_default_transform(src.crs, dst_crs, src.width, src.height, *src.bounds)
- kwargs = src.meta.copy()
- kwargs.update({"crs": dst_crs, "transform": transform, "width": width, "height": height})
-
- with NamedTemporaryFile(suffix=".tif", delete=False) as projected_file:
- with rasterio.open(projected_file.name, "w", **kwargs) as dst:
- for i in range(1, src.count + 1):
- reproject(
- source=rasterio.band(src, i),
- destination=rasterio.band(dst, i),
- src_transform=src.transform,
- src_crs=src.crs,
- dst_transform=transform,
- dst_crs=dst_crs,
- resampling=Resampling.average,
- resolution=resolution,
- )
- return projected_file.name
+ crs_obj = CRS(crs)
+ ellps = crs_obj.geodetic_crs.ellipsoid
+ a = ellps.semi_major_metre
+ inv_f = ellps.inverse_flattening
+ f = 1.0 / inv_f if inv_f and not math.isinf(inv_f) else 0.0
+ e2 = 2.0 * f - f * f
+
+ to_geo = Transformer.from_crs(crs_obj, "EPSG:4326", always_xy=True)
+ from_geo = Transformer.from_crs("EPSG:4326", crs_obj, always_xy=True)
+
+ lon, lat = to_geo.transform(x, y)
+
+ # Numerical Jacobian d(crs) / d(lon, lat) in m_crs / deg.
+ xp, yp = from_geo.transform(np.add(lon, eps_deg), lat)
+ xm, ym = from_geo.transform(np.subtract(lon, eps_deg), lat)
+ xyp, yyp = from_geo.transform(lon, np.add(lat, eps_deg))
+ xym, yym = from_geo.transform(lon, np.subtract(lat, eps_deg))
+
+ j_xlon = (np.asarray(xp) - np.asarray(xm)) / (2.0 * eps_deg)
+ j_ylon = (np.asarray(yp) - np.asarray(ym)) / (2.0 * eps_deg)
+ j_xlat = (np.asarray(xyp) - np.asarray(xym)) / (2.0 * eps_deg)
+ j_ylat = (np.asarray(yyp) - np.asarray(yym)) / (2.0 * eps_deg)
+
+ # Convert deg → true-metres using the local radii of curvature.
+ sin_lat = np.sin(np.radians(lat))
+ cos_lat = np.cos(np.radians(lat))
+ w = np.sqrt(1.0 - e2 * sin_lat * sin_lat)
+ n_prime = a / w # prime-vertical radius (m)
+ m_meridional = a * (1.0 - e2) / w**3 # meridional radius (m)
+ deg_to_rad = math.pi / 180.0
+ m_per_deg_lon = n_prime * cos_lat * deg_to_rad
+ m_per_deg_lat = m_meridional * deg_to_rad
+
+ # det of the Jacobian in (m_crs / m_true)^2.
+ det = np.abs(j_xlon * j_ylat - j_xlat * j_ylon) / (m_per_deg_lon * m_per_deg_lat)
+
+ k = np.sqrt(det)
+ return float(k) if np.isscalar(x) and np.isscalar(y) else k
diff --git a/pism_terra/sampling.py b/pism_terra/sampling.py
index ce88e69..487ac39 100644
--- a/pism_terra/sampling.py
+++ b/pism_terra/sampling.py
@@ -131,34 +131,34 @@ def _make_frozen(dist_name: str, spec: dict[str, Any]):
return dist(*args, loc=loc, scale=scale)
-def create_samples(d: dict[str, dict[str, Any]], n_samples: int = 10, seed: int | None = None) -> pd.DataFrame:
+def _transform_quantiles(U: np.ndarray, d: dict[str, dict[str, Any]]) -> pd.DataFrame:
"""
- Draw Latin Hypercube samples and transform by the specified SciPy distributions.
+ Map unit-cube quantiles to parameter values via each variable's inverse CDF.
+
+ Shared back end for :func:`create_samples` (LHS quantiles) and
+ :func:`create_grid_samples` (regular-grid quantiles): both produce a matrix
+ ``U`` of quantiles in ``[0, 1]`` and differ only in how those quantiles are
+ laid out.
Parameters
----------
+ U : numpy.ndarray
+ Array of shape ``(n, len(d))`` of quantiles in ``[0, 1]``; column ``i``
+ corresponds to the ``i``-th variable in ``d`` (insertion order).
d : dict
- Mapping ``name -> spec`` where spec includes at least ``distribution`` and any
- required parameters (e.g., ``low, high`` for ``randint`` or ``loc, scale`` for
- continuous distributions; plus any shapes like ``a, b`` for ``truncnorm``).
- n_samples : int, default 10
- Number of samples to draw.
- seed : int or None, default None
- Seed for the Latin Hypercube sampler.
+ Mapping ``name -> spec`` where spec includes at least ``distribution``
+ and any required parameters.
Returns
-------
pandas.DataFrame
- A DataFrame with one column per variable in ``d`` and an
- integer ``sample`` column. Discrete variables are returned as integer dtype.
+ One column per variable in ``d`` plus an integer ``sample`` column.
+ Discrete variables are returned as integer dtype.
"""
names = list(d.keys())
- # LHS in [0,1]
- engine = qmc.LatinHypercube(d=len(names), seed=seed)
- U = engine.random(n_samples)
-
- # clip away exactly 0 or 1 so ppf is well-defined
+ # clip away exactly 0 or 1 so ppf is well-defined (ppf(1) is +inf for
+ # unbounded distributions)
eps = np.finfo(float).eps # pylint: disable=no-member
U = np.clip(U, eps, 1 - eps)
@@ -186,5 +186,136 @@ def create_samples(d: dict[str, dict[str, Any]], n_samples: int = 10, seed: int
df[col] = df[col].round().astype("int64")
# add sample id
- df.insert(0, "sample", np.arange(n_samples, dtype=int))
+ df.insert(0, "sample", np.arange(len(df), dtype=int))
return df
+
+
+def create_samples(d: dict[str, dict[str, Any]], n_samples: int = 10, seed: int | None = None) -> pd.DataFrame:
+ """
+ Draw Latin Hypercube samples and transform by the specified SciPy distributions.
+
+ Parameters
+ ----------
+ d : dict
+ Mapping ``name -> spec`` where spec includes at least ``distribution`` and any
+ required parameters (e.g., ``low, high`` for ``randint`` or ``loc, scale`` for
+ continuous distributions; plus any shapes like ``a, b`` for ``truncnorm``).
+ n_samples : int, default 10
+ Number of samples to draw.
+ seed : int or None, default None
+ Seed for the Latin Hypercube sampler.
+
+ Returns
+ -------
+ pandas.DataFrame
+ A DataFrame with one column per variable in ``d`` and an
+ integer ``sample`` column. Discrete variables are returned as integer dtype.
+ """
+ names = list(d.keys())
+
+ # LHS in [0,1]
+ engine = qmc.LatinHypercube(d=len(names), seed=seed)
+ U = engine.random(n_samples)
+
+ return _transform_quantiles(U, d)
+
+
+def create_grid_samples(d: dict[str, dict[str, Any]], n_levels: int = 10, kind: str = "edge") -> pd.DataFrame:
+ """
+ Build a full-factorial (regular grid) design and transform by the SciPy distributions.
+
+ Unlike :func:`create_samples` (a randomized Latin Hypercube), this lays each
+ variable on an equidistant grid of quantiles and takes the **Cartesian
+ product** across variables, so the result has ``n_levels ** len(d)`` rows.
+ For a single variable this is just ``n_levels`` equidistant draws.
+
+ Parameters
+ ----------
+ d : dict
+ Mapping ``name -> spec`` (same structure as :func:`create_samples`).
+ n_levels : int, default 10
+ Number of equidistant levels **per variable**. Total rows are
+ ``n_levels ** len(d)``.
+ kind : {"edge", "midpoint", "endpoint"}, default "edge"
+ Quantile convention for the per-variable grid of ``n`` levels:
+
+ - ``"edge"``: ``i / n`` for ``i = 1..n`` (e.g. ``1/3, 2/3, 3/3``). Reaches
+ the distribution's upper bound; for *unbounded* distributions the top
+ level is an extreme (clipped) value, so prefer ``"midpoint"`` there.
+ - ``"midpoint"``: ``(i + 0.5) / n`` for ``i = 0..n-1`` (e.g.
+ ``1/6, 1/2, 5/6``). Symmetric; never hits a bound.
+ - ``"endpoint"``: ``linspace(0, 1, n)`` (e.g. ``0, 1/2, 1``). Includes
+ both bounds.
+
+ Returns
+ -------
+ pandas.DataFrame
+ A DataFrame with one column per variable in ``d`` and an integer
+ ``sample`` column. Discrete variables are returned as integer dtype.
+
+ Raises
+ ------
+ ValueError
+ If ``n_levels < 1`` or ``kind`` is not recognized.
+ """
+ if n_levels < 1:
+ raise ValueError(f"n_levels must be >= 1, got {n_levels}")
+
+ names = list(d.keys())
+
+ if kind == "edge":
+ q = np.arange(1, n_levels + 1) / n_levels
+ elif kind == "midpoint":
+ q = (np.arange(n_levels) + 0.5) / n_levels
+ elif kind == "endpoint":
+ q = np.linspace(0.0, 1.0, n_levels)
+ else:
+ raise ValueError(f"unknown kind '{kind}'; use 'edge', 'midpoint', or 'endpoint'")
+
+ # Cartesian product of the per-variable grids -> (n_levels ** d, d)
+ grids = np.meshgrid(*([q] * len(names)), indexing="ij")
+ U = np.stack([g.ravel() for g in grids], axis=1)
+
+ return _transform_quantiles(U, d)
+
+
+def generate_samples(
+ d: dict[str, dict[str, Any]],
+ n_samples: int = 10,
+ method: str | None = None,
+ seed: int | None = None,
+) -> pd.DataFrame:
+ """
+ Dispatch to a sampling strategy by name.
+
+ Parameters
+ ----------
+ d : dict
+ Mapping ``name -> spec`` (same structure as :func:`create_samples`).
+ n_samples : int, default 10
+ For ``method="lhs"`` the total number of draws; for ``method="factorial"``
+ the number of equidistant levels **per variable** (total rows are
+ ``n_samples ** len(d)``).
+ method : str or None, default None
+ Sampling strategy. ``None`` or ``"lhs"`` (and aliases) selects the
+ Latin Hypercube; ``"factorial"`` (and aliases ``"grid"``,
+ ``"full_factorial"``) selects the full-factorial grid. Case-insensitive.
+ seed : int or None, default None
+ Seed for the Latin Hypercube sampler (ignored for the deterministic grid).
+
+ Returns
+ -------
+ pandas.DataFrame
+ Sample table as returned by the selected sampler.
+
+ Raises
+ ------
+ ValueError
+ If ``method`` is not recognized.
+ """
+ m = (method or "lhs").strip().lower()
+ if m in {"lhs", "latin", "latin_hypercube", "latinhypercube"}:
+ return create_samples(d, n_samples=n_samples, seed=seed)
+ if m in {"factorial", "grid", "full_factorial"}:
+ return create_grid_samples(d, n_levels=n_samples)
+ raise ValueError(f"unknown sampling method '{method}'; use 'lhs' or 'factorial'")
diff --git a/pism_terra/templates/anvil.j2 b/pism_terra/templates/anvil.j2
deleted file mode 100644
index b2d9b24..0000000
--- a/pism_terra/templates/anvil.j2
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/sh
-#SBATCH -A ees240003
-#SBATCH --ntasks={{ ntasks }}
-#SBATCH --nodes={{ nodes }}
-#SBATCH --time={{ walltime }}
-#SBATCH -p {{ queue }}
-#SBATCH --mail-type=BEGIN
-#SBATCH --mail-type=END
-#SBATCH --mail-type=FAIL
-#SBATCH --output=pism.%j
-
-module list
-
-umask 007
-
-cd $SLURM_SUBMIT_DIR
-
-# Generate a list of compute node hostnames reserved for this job,
-# this ./nodes file is necessary for slurm to spawn mpi processes
-# across multiple compute nodes
-srun -l /bin/hostname | sort -n | awk '{print $2}' > ./nodes_$SLURM_JOBID
-
-ulimit -l unlimited
-ulimit -s unlimited
-ulimit
diff --git a/pism_terra/templates/chinook-apptainer-async.j2 b/pism_terra/templates/chinook-apptainer-async.j2
new file mode 100644
index 0000000..c2ae958
--- /dev/null
+++ b/pism_terra/templates/chinook-apptainer-async.j2
@@ -0,0 +1,31 @@
+#!/bin/sh
+#SBATCH --partition={{ queue | default("t2small") }}
+#SBATCH --ntasks={{ ntasks | default(40) }}
+#SBATCH --tasks-per-node={{ tasks | default(40) }}
+#SBATCH --time={{ walltime | default("24:00:00") }}
+#SBATCH --mail-type=BEGIN
+#SBATCH --mail-type=END
+#SBATCH --mail-type=FAIL
+#SBATCH --output="{{ output_path }}/pism.%j"
+
+module purge
+module load intel-compilers/2023.1.0 impi/2021.9.0
+
+export container=$HOME/pism-dev.sif
+if [ ! -f "${container}" ]; then
+ echo "Container ${container} not found." >&2
+ exit 1
+fi
+echo "Using container ${container}"
+
+umask 007
+
+cd $SLURM_SUBMIT_DIR
+
+ulimit -l unlimited
+ulimit -s unlimited
+ulimit
+
+pismtasks=$(( SLURM_NTASKS - 4 ))
+
+mpiexec -n 1 apptainer run ${container} pism_async_writer -r 1 : -n ${pismtasks} apptainer run ${container} pism {{ run_str }} -output.asynchronous
diff --git a/pism_terra/templates/chinook-apptainer-inverse.j2 b/pism_terra/templates/chinook-apptainer-inverse.j2
new file mode 100644
index 0000000..246e795
--- /dev/null
+++ b/pism_terra/templates/chinook-apptainer-inverse.j2
@@ -0,0 +1,47 @@
+#!/bin/sh
+#SBATCH --partition={{ queue | default("t2small") }}
+#SBATCH --ntasks={{ ntasks | default(40) }}
+#SBATCH --tasks-per-node={{ tasks | default(40) }}
+#SBATCH --time={{ walltime | default("24:00:00") }}
+#SBATCH --mail-type=BEGIN
+#SBATCH --mail-type=END
+#SBATCH --mail-type=FAIL
+#SBATCH --output="{{ output_path }}/pism.%j"
+
+module purge
+module load intel-compilers/2023.1.0 iimpi/2023a
+
+export container=$HOME/pism-ismip7.sif
+if [ ! -f "${container}" ]; then
+ echo "Container ${container} not found." >&2
+ exit 1
+fi
+echo "Using container ${container}"
+
+umask 007
+
+cd $SLURM_SUBMIT_DIR
+
+ulimit -l unlimited
+ulimit -s unlimited
+ulimit
+
+conda deactivate
+
+# ###############################################################################
+# FORWARD RUN
+# ###############################################################################
+
+mpirun -np {{ ntasks | default(40) }} apptainer run --cleanenv ${container} pism {{ run_str }}
+
+# ###############################################################################
+# INVERSION RUN
+# ###############################################################################
+
+mpirun -np {{ ntasks | default(40) }} apptainer run --cleanenv ${container} pismi {{ inv_str }}
+
+# ###############################################################################
+# Postprocessing
+# ###############################################################################
+
+{{ post_script }} --ntasks {{ ntasks | default(40) }} {{ post_file }}
diff --git a/pism_terra/templates/chinook-apptainer-ismip7-async.j2 b/pism_terra/templates/chinook-apptainer-ismip7-async.j2
new file mode 100644
index 0000000..ab75cb7
--- /dev/null
+++ b/pism_terra/templates/chinook-apptainer-ismip7-async.j2
@@ -0,0 +1,39 @@
+#!/bin/sh
+#SBATCH --partition={{ queue | default("t2small") }}
+#SBATCH --ntasks={{ ntasks | default(40) }}
+#SBATCH --tasks-per-node={{ tasks | default(40) }}
+#SBATCH --time={{ walltime | default("48:00:00") }}
+#SBATCH --mail-type=BEGIN
+#SBATCH --mail-type=END
+#SBATCH --mail-type=FAIL
+#SBATCH --output="{{ output_path }}/pism.%j"
+
+module purge
+module load intel-compilers/2023.1.0 impi/2021.9.0
+
+export container=$HOME/pism-ismip7.sif
+if [ ! -f "${container}" ]; then
+ echo "Container ${container} not found." >&2
+ exit 1
+fi
+echo "Using container ${container}"
+
+umask 007
+
+cd $SLURM_SUBMIT_DIR
+
+ulimit -l unlimited
+ulimit -s unlimited
+ulimit
+
+pismtasks=$(( SLURM_NTASKS - 4 ))
+
+# ###############################################################################
+# FORWARD RUN
+# ###############################################################################
+
+mpiexec -n 1 apptainer run ${container} pism_async_writer : -n ${pismtasks} apptainer run ${container} pism {{ run_hist_str }} -output.asynchronous
+
+{% if run_proj_str %}
+mpiexec -n 1 apptainer run ${container} pism_async_writer : -n ${pismtasks} apptainer run ${container} pism {{ run_proj_str }} -output.asynchronous
+{% endif %}
diff --git a/pism_terra/templates/chinook-apptainer-ismip7.j2 b/pism_terra/templates/chinook-apptainer-ismip7.j2
new file mode 100644
index 0000000..039ba4b
--- /dev/null
+++ b/pism_terra/templates/chinook-apptainer-ismip7.j2
@@ -0,0 +1,47 @@
+#!/bin/sh
+#SBATCH --partition={{ queue | default("t2small") }}
+#SBATCH --ntasks={{ ntasks | default(40) }}
+#SBATCH --tasks-per-node={{ tasks | default(40) }}
+#SBATCH --time={{ walltime | default("48:00:00") }}
+#SBATCH --mail-type=BEGIN
+#SBATCH --mail-type=END
+#SBATCH --mail-type=FAIL
+#SBATCH --output="{{ output_path }}/pism.%j"
+
+module purge
+module load intel-compilers/2023.1.0 impi/2021.9.0
+
+export container=$HOME/pism-ismip7.sif
+if [ ! -f "${container}" ]; then
+ echo "Container ${container} not found." >&2
+ exit 1
+fi
+echo "Using container ${container}"
+
+umask 007
+
+cd $SLURM_SUBMIT_DIR
+
+ulimit -l unlimited
+ulimit -s unlimited
+ulimit
+
+pismtasks=$(( SLURM_NTASKS - 4 ))
+
+# ###############################################################################
+# FORWARD RUN
+# ###############################################################################
+
+mpiexec -n 1 apptainer run ${container} pism_ismip7_writer -r 1 : -n ${pismtasks} apptainer run ${container} pism {{ run_hist_str }} -output.asynchronous
+
+{% if run_proj_str %}
+mpiexec -n 1 apptainer run ${container} pism_ismip7_writer -r 1 : -n ${pismtasks} apptainer run ${container} pism {{ run_proj_str }} -output.asynchronous
+{% endif %}
+
+{{ post_scalar_str }}
+
+# ###############################################################################
+# COMPLIANCE CHECKER
+# ###############################################################################
+
+{{ ism_checker_str }}
diff --git a/pism_terra/templates/chinook-apptainer-kitp.j2 b/pism_terra/templates/chinook-apptainer-kitp.j2
new file mode 100644
index 0000000..7c45a1a
--- /dev/null
+++ b/pism_terra/templates/chinook-apptainer-kitp.j2
@@ -0,0 +1,31 @@
+#!/bin/sh
+#SBATCH --partition={{ queue | default("t2small") }}
+{# +1 for the pism-kitp-writer co-process launched via MPMD below #}
+#SBATCH --ntasks={{ ntasks | default(36) + 1 }}
+#SBATCH --tasks-per-node={{ tasks | default(40) }}
+#SBATCH --time={{ walltime | default("24:00:00") }}
+#SBATCH --mail-type=BEGIN
+#SBATCH --mail-type=END
+#SBATCH --mail-type=FAIL
+#SBATCH --output="{{ output_path }}/pism.%j"
+
+module purge
+module load intel-compilers/2023.1.0 iimpi/2023a
+
+export container=$HOME/pism-dev.sif
+if [ ! -f "${container}" ]; then
+ echo "Container ${container} not found." >&2
+ exit 1
+fi
+echo "Using container ${container}"
+
+umask 007
+
+cd $SLURM_SUBMIT_DIR
+
+ulimit -l unlimited
+ulimit -s unlimited
+ulimit
+
+
+mpirun -np {{ ntasks | default(36) }} apptainer run ${container} pism {{ run_str }} : -n 1 apptainer run ${container} pism_kitp_writer -d -g {{ geometry_file }}
diff --git a/pism_terra/templates/chinook-apptainer.j2 b/pism_terra/templates/chinook-apptainer.j2
new file mode 100644
index 0000000..438377a
--- /dev/null
+++ b/pism_terra/templates/chinook-apptainer.j2
@@ -0,0 +1,33 @@
+#!/bin/sh
+#SBATCH --partition={{ queue | default("t2small") }}
+#SBATCH --ntasks={{ ntasks | default(40) }}
+#SBATCH --tasks-per-node={{ tasks | default(40) }}
+#SBATCH --time={{ walltime | default("48:00:00") }}
+#SBATCH --mail-type=BEGIN
+#SBATCH --mail-type=END
+#SBATCH --mail-type=FAIL
+#SBATCH --output="{{ output_path }}/pism.%j"
+
+module purge
+module load intel-compilers/2023.1.0 impi/2021.9.0
+
+export container=$HOME/pism-ismip7.sif
+if [ ! -f "${container}" ]; then
+ echo "Container ${container} not found." >&2
+ exit 1
+fi
+echo "Using container ${container}"
+
+umask 007
+
+cd $SLURM_SUBMIT_DIR
+
+ulimit -l unlimited
+ulimit -s unlimited
+ulimit
+
+# ###############################################################################
+# FORWARD RUN
+# ###############################################################################
+
+mpiexec -n $SLURM_NTASKS apptainer run ${container} pism {{ run_str }}
diff --git a/pism_terra/templates/chinook-ismip7.j2 b/pism_terra/templates/chinook-ismip7.j2
new file mode 100644
index 0000000..08b1f5f
--- /dev/null
+++ b/pism_terra/templates/chinook-ismip7.j2
@@ -0,0 +1,39 @@
+#!/bin/sh
+#SBATCH --partition={{ queue | default("t2small") }}
+#SBATCH --ntasks={{ ntasks | default(40) }}
+#SBATCH --tasks-per-node={{ tasks | default(40) }}
+#SBATCH --time={{ walltime | default("24:00:00") }}
+#SBATCH --mail-type=BEGIN
+#SBATCH --mail-type=END
+#SBATCH --mail-type=FAIL
+#SBATCH --output="{{ output_path }}/pism.%j"
+
+umask 007
+
+cd $SLURM_SUBMIT_DIR
+
+ulimit -l unlimited
+ulimit -s unlimited
+ulimit
+
+# ###############################################################################
+# FORWARD RUN
+# ###############################################################################
+
+mpirun -np {{ ntasks | default(40) }} pism {{ run_hist_str }}
+
+{% if run_proj_str %}
+mpirun -np {{ ntasks | default(40) }} pism {{ run_proj_str }}
+{% endif %}
+
+# ###############################################################################
+# SCALAR POST-PROCESSING
+# ###############################################################################
+
+{{ post_scalar_str }}
+
+# ###############################################################################
+# COMPLIANCE CHECKER
+# ###############################################################################
+
+{{ ism_checker_str }}
diff --git a/pism_terra/templates/chinook.j2 b/pism_terra/templates/chinook.j2
index 55f0a37..35fe839 100644
--- a/pism_terra/templates/chinook.j2
+++ b/pism_terra/templates/chinook.j2
@@ -1,23 +1,23 @@
#!/bin/sh
-#SBATCH --partition={{ queue }}
-#SBATCH --ntasks={{ ntasks }}
-#SBATCH --tasks-per-node=40
-#SBATCH --time={{ walltime }}
+#SBATCH --partition={{ queue | default("t2small") }}
+#SBATCH --ntasks={{ ntasks | default(40) }}
+#SBATCH --tasks-per-node={{ tasks | default(40) }}
+#SBATCH --time={{ walltime | default("24:00:00") }}
#SBATCH --mail-type=BEGIN
#SBATCH --mail-type=END
#SBATCH --mail-type=FAIL
#SBATCH --output="{{ output_path }}/pism.%j"
-module list
-
-conda deactivate
-
umask 007
cd $SLURM_SUBMIT_DIR
-srun -l /bin/hostname | sort -n | awk '{print $2}' > ./nodes_$SLURM_JOBID
-
ulimit -l unlimited
ulimit -s unlimited
ulimit
+
+# ###############################################################################
+# FORWARD RUN
+# ###############################################################################
+
+mpirun -np {{ ntasks | default(40) }} pism {{ run_str }}
diff --git a/pism_terra/templates/debug-inverse.j2 b/pism_terra/templates/debug-inverse.j2
new file mode 100644
index 0000000..b980fbf
--- /dev/null
+++ b/pism_terra/templates/debug-inverse.j2
@@ -0,0 +1,5 @@
+mpirun -np {{ ntasks | default(8) }} pism {{ run_str }}
+
+mpirun -np {{ ntasks | default(8) }} pismi {{ inv_str }}
+
+{{ post_script }} --ntasks {{ ntasks | default(8) }} {{ post_file }}
diff --git a/pism_terra/templates/debug-ismip7.j2 b/pism_terra/templates/debug-ismip7.j2
new file mode 100644
index 0000000..be319a2
--- /dev/null
+++ b/pism_terra/templates/debug-ismip7.j2
@@ -0,0 +1,9 @@
+mpirun -np {{ ntasks | default(8) }} pism {{ run_hist_str }}
+
+{{ ism_checker_str }}
+
+{% if run_proj_str %}
+mpirun -np {{ ntasks | default(8) }} pism {{ run_proj_str }}
+{% endif %}
+
+{{ post_scalar_str }}
diff --git a/pism_terra/templates/debug-kitp-async.j2 b/pism_terra/templates/debug-kitp-async.j2
new file mode 100644
index 0000000..a88c804
--- /dev/null
+++ b/pism_terra/templates/debug-kitp-async.j2
@@ -0,0 +1 @@
+mpirun -np {{ ntasks | default(8) }} pism {{ run_str }} : -n 1 pism-kitp-writer -d -g {{ geometry_file }} -r 2
diff --git a/pism_terra/templates/debug.j2 b/pism_terra/templates/debug.j2
index e69de29..845e4eb 100644
--- a/pism_terra/templates/debug.j2
+++ b/pism_terra/templates/debug.j2
@@ -0,0 +1,3 @@
+mpirun -np {{ ntasks | default(8) }} pism {{ run_str }}
+
+{{ post_script }} --ntasks {{ ntasks | default(8) }} {{ post_file }}
diff --git a/pism_terra/templates/ec2-inverse.j2 b/pism_terra/templates/ec2-inverse.j2
new file mode 100644
index 0000000..102008c
--- /dev/null
+++ b/pism_terra/templates/ec2-inverse.j2
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+ulimit -a
+
+mpirun -np {{ ntasks | default(32) }} pism {{ run_str }}
+
+mpirun -np {{ ntasks | default(32) }} /opt/pism/bin/python -m PISM.pismi {{ inv_str }}
diff --git a/pism_terra/templates/ec2.j2 b/pism_terra/templates/ec2.j2
index 1c07a3f..41fb760 100644
--- a/pism_terra/templates/ec2.j2
+++ b/pism_terra/templates/ec2.j2
@@ -1,3 +1,7 @@
#!/usr/bin/env bash
ulimit -a
+
+mpirun -np {{ ntasks | default(32) }} pism {{ run_str }}
+
+{{ ism_checker_str }}
diff --git a/pism_terra/templates/stampede3.j2 b/pism_terra/templates/stampede3.j2
deleted file mode 100644
index 0d09c6f..0000000
--- a/pism_terra/templates/stampede3.j2
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh
-#SBATCH -n {{ ntasks }}
-#SBATCH -N {{ nodes }}
-#SBATCH --time={{ walltime }}
-#SBATCH -p {{ queue }}
-#SBATCH --mail-type=BEGIN
-#SBATCH --mail-type=END
-#SBATCH --mail-type=FAIL
-#SBATCH --output="{{ output_path }}/pism.%j"
-
-module list
-
-umask 007
-
-cd $SLURM_SUBMIT_DIR
-
-ulimit -l unlimited
-ulimit -s unlimited
-ulimit
diff --git a/pism_terra/templates/trillian.j2 b/pism_terra/templates/trillian.j2
deleted file mode 100644
index 1c6b39f..0000000
--- a/pism_terra/templates/trillian.j2
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/sh
-#SBATCH --partition={{ queue }}
-#SBATCH --ntasks={{ ntasks }}
-#SBATCH --tasks-per-node=16
-#SBATCH --time={{ walltime }}
-#SBATCH --mail-type=BEGIN
-#SBATCH --mail-type=END
-#SBATCH --mail-type=FAIL
-#SBATCH --output="{{ output_path }}/pism.%j"
-
-module list
-
-umask 007
-
-cd $SLURM_SUBMIT_DIR
-
-srun -l /bin/hostname | sort -n | awk '{print $2}' > ./nodes_$SLURM_JOBID
-
-ulimit -l unlimited
-ulimit -s unlimited
-ulimit
diff --git a/pism_terra/uq/calib.toml b/pism_terra/uq/calib.toml
index f36681f..5821b50 100644
--- a/pism_terra/uq/calib.toml
+++ b/pism_terra/uq/calib.toml
@@ -1,4 +1,4 @@
-samples = 250
+samples = 500
['basal_resistance.pseudo_plastic.q']
@@ -20,8 +20,8 @@ distribution = "uniform"
['basal_yield_stress.mohr_coulomb.till_phi_default']
-loc = 5
-scale = 40
+loc = 15
+scale = 30
distribution = "uniform"
['stress_balance.blatter.enhancement_factor']
diff --git a/pism_terra/uq/era5_pdd.toml b/pism_terra/uq/era5_pdd.toml
index a84b8d6..2e04dc7 100644
--- a/pism_terra/uq/era5_pdd.toml
+++ b/pism_terra/uq/era5_pdd.toml
@@ -1,4 +1,4 @@
-samples = 9
+bsamples = 9
mapping = {"atmosphere.given.file"= "climate_file", "surface.given.file"= "climate_file"}
['surface.pdd.factor_ice']
diff --git a/pism_terra/uq/ismip7_historical_calib.toml b/pism_terra/uq/ismip7_historical_calib.toml
new file mode 100644
index 0000000..8b99563
--- /dev/null
+++ b/pism_terra/uq/ismip7_historical_calib.toml
@@ -0,0 +1,7 @@
+samples = 20
+
+['calving.vonmises_calving.sigma_max']
+
+loc = 400000
+scale = 400000
+distribution = "uniform"
diff --git a/pism_terra/uq/kitp.toml b/pism_terra/uq/kitp.toml
new file mode 100644
index 0000000..d6a2d02
--- /dev/null
+++ b/pism_terra/uq/kitp.toml
@@ -0,0 +1,14 @@
+samples = 25
+mapping = {"atmosphere.given.file"= "climate_file", "surface.given.file"= "climate_file"}
+
+['surface.debm_simple.c1']
+
+loc = 25
+scale = 10
+distribution = "uniform"
+
+['surface.debm_simple.c2']
+
+loc = -130
+scale = 10
+distribution = "uniform"
diff --git a/pism_terra/uq/kitp_debm.toml b/pism_terra/uq/kitp_debm.toml
new file mode 100644
index 0000000..97b550c
--- /dev/null
+++ b/pism_terra/uq/kitp_debm.toml
@@ -0,0 +1,32 @@
+samples = 150
+mapping = {"atmosphere.given.file"= "climate_file", "surface.given.file"= "climate_file"}
+
+['surface.debm_simple.c1']
+
+loc = 5
+scale = 45
+distribution = "uniform"
+
+['surface.debm_simple.c2']
+
+loc = -130
+scale = 100
+distribution = "uniform"
+
+['surface.debm_simple.air_temp_all_precip_as_snow']
+
+loc = 271.15
+scale = 1
+distribution = "uniform"
+
+['surface.debm_simple.air_temp_all_precip_as_rain']
+
+loc = 273.15
+scale = 4
+distribution = "uniform"
+
+['surface.debm_simple.refreeze']
+
+loc = 0.2
+scale = 0.6
+distribution = "uniform"
diff --git a/pism_terra/uq/kitp_pdd.toml b/pism_terra/uq/kitp_pdd.toml
new file mode 100644
index 0000000..6e3ba8e
--- /dev/null
+++ b/pism_terra/uq/kitp_pdd.toml
@@ -0,0 +1,24 @@
+samples = 75
+mapping = {"atmosphere.given.file"= "climate_file", "surface.given.file"= "climate_file"}
+
+['surface.pdd.factor_ice']
+
+loc = 0.008791208791208791
+scale = 0.004395604395604396
+a = -1
+b = 1
+distribution = "truncnorm"
+
+['surface.pdd.factor_snow']
+
+loc = 0.004395604395604396
+scale = 0.0016483516483516484
+a = -1.2
+b = 1.2
+distribution = "truncnorm"
+
+['surface.pdd.refreeze']
+
+loc = 0.2
+scale = 0.6
+distribution = "uniform"
diff --git a/pism_terra/uq/wrangell.toml b/pism_terra/uq/wrangell.toml
index 6352902..51e8f9f 100644
--- a/pism_terra/uq/wrangell.toml
+++ b/pism_terra/uq/wrangell.toml
@@ -1,5 +1,11 @@
samples = 9
-mapping = {"atmosphere.given.file"= "climate_file", "surface.given.file"= "climate_file"}
+mapping = {}
+
+['atmosphere.elevation_change.temperature_lapse_rate']
+
+loc = 5
+scale = 2
+distribution = "uniform"
['atmosphere.delta_T']
@@ -16,7 +22,7 @@ distribution = "randint"
['atmosphere.precip_exponential_factor_for_temperature']
loc = 0.05
-scale = 0.03
+scale = 0.04
distribution = "uniform"
['surface.pdd.factor_ice']
diff --git a/pism_terra/validate.py b/pism_terra/validate.py
new file mode 100644
index 0000000..ed6712f
--- /dev/null
+++ b/pism_terra/validate.py
@@ -0,0 +1,184 @@
+# Copyright (C) 2026 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+"""
+Bulk-validate NetCDF files under a directory using :func:`check_xr_lazy`.
+"""
+
+from __future__ import annotations
+
+from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+from typing import Sequence
+
+import xarray as xr
+from tqdm.auto import tqdm
+
+from pism_terra.workflow import check_dataset_lazy
+
+
+def _validate_one(path_str: str) -> tuple[bool, str]:
+ """
+ Open a dataset, run the sampled health check, and surface any failure reason.
+
+ Mirrors :func:`pism_terra.workflow.check_xr_lazy` but returns the failure
+ reason instead of printing it, so the parent process can show it both
+ inline and in the final summary.
+
+ Parameters
+ ----------
+ path_str : str
+ Absolute path to the NetCDF / Zarr dataset (str rather than ``Path``
+ keeps worker process pickling cheap).
+
+ Returns
+ -------
+ tuple of (bool, str)
+ ``(True, "")`` on success. ``(False, "ExceptionType: message")`` if
+ the dataset cannot be opened or fails the lazy checks.
+ """
+ try:
+ time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
+ delta_coder = xr.coders.CFTimedeltaCoder()
+ ds = xr.open_dataset(path_str, decode_times=time_coder, decode_timedelta=delta_coder)
+ check_dataset_lazy(ds)
+ except FileNotFoundError as exc:
+ return False, f"FileNotFoundError: {exc}"
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ return False, f"{type(exc).__name__}: {exc}"
+ return True, ""
+
+
+def validate_directory(
+ directory: Path | str,
+ workers: int = 8,
+ pattern: str = "*.nc",
+) -> list[tuple[Path, str]]:
+ """
+ Validate every NetCDF under *directory* in parallel.
+
+ Walks *directory* recursively for files matching *pattern*, runs the
+ sampled health check on each in a worker process pool, and prints both
+ inline per-file failures and an end-of-run summary that includes the
+ failure reason (exception type + message) for each invalid file.
+
+ Parameters
+ ----------
+ directory : pathlib.Path or str
+ Root directory to search. Searched recursively via ``Path.rglob``.
+ workers : int, default 8
+ Number of parallel worker processes. HDF5 is not reliably
+ thread-safe across builds, so each worker gets its own interpreter
+ and HDF5 state.
+ pattern : str, default ``"*.nc"``
+ Glob pattern matched against file names during the recursive walk.
+
+ Returns
+ -------
+ list of tuple of (pathlib.Path, str)
+ One entry per failed file: ``(path, "ExceptionType: message")``.
+ Empty when everything passes. The previous version returned only
+ the paths; the reason string is added so callers can present it in
+ their own reports without re-opening the file.
+ """
+ root = Path(directory)
+ if not root.is_dir():
+ raise NotADirectoryError(f"{root} is not a directory")
+
+ files = sorted(root.rglob(pattern))
+ if not files:
+ print(f"No files matching {pattern!r} under {root}")
+ return []
+
+ invalid: list[tuple[Path, str]] = []
+ # Processes (not threads): HDF5 isn't reliably thread-safe across all
+ # builds (Chinook's HPC build segfaults under concurrent opens), so
+ # each worker gets its own interpreter and HDF5 state.
+ with ProcessPoolExecutor(max_workers=workers) as executor:
+ future_to_path = {executor.submit(_validate_one, str(p)): p for p in files}
+ for future in tqdm(
+ as_completed(future_to_path),
+ total=len(future_to_path),
+ desc=f"Validating {root}",
+ unit="file",
+ ):
+ p = future_to_path[future]
+ try:
+ ok, reason = future.result()
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ # _validate_one catches its own exceptions, so this is defensive.
+ ok = False
+ reason = f"{type(exc).__name__}: {exc}"
+ if not ok:
+ invalid.append((p, reason))
+ # Surface inline so the user sees failures as the bar advances.
+ print(f"{p.resolve()} is not valid ✗ ({reason})")
+
+ return invalid
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ """
+ Run the bulk NetCDF validator.
+
+ Parameters
+ ----------
+ argv : sequence of str or None, optional
+ Command-line arguments (without the program name). When ``None``
+ (default), :data:`sys.argv` is used.
+
+ Returns
+ -------
+ int
+ Process exit code: ``0`` when every file is valid (or no files were
+ found), ``1`` when at least one file failed validation.
+ """
+ parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+ parser.description = "Recursively validate NetCDF files under a directory."
+ parser.add_argument(
+ "--workers",
+ help="Number of parallel worker threads.",
+ type=int,
+ default=8,
+ )
+ parser.add_argument(
+ "--pattern",
+ help="Glob pattern for file names (matched recursively).",
+ type=str,
+ default="*.nc",
+ )
+ parser.add_argument(
+ "DIRECTORY",
+ help="Directory to walk recursively.",
+ )
+ args = parser.parse_args(list(argv) if argv is not None else None)
+
+ invalid = validate_directory(args.DIRECTORY, workers=args.workers, pattern=args.pattern)
+
+ if invalid:
+ print(f"\n{len(invalid)} file(s) failed validation:")
+ for p, reason in invalid:
+ print(f" {p.resolve()}")
+ print(f" → {reason}")
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ __spec__ = None # type: ignore
+ raise SystemExit(main())
diff --git a/pism_terra/vector.py b/pism_terra/vector.py
index cfff965..1dcc432 100644
--- a/pism_terra/vector.py
+++ b/pism_terra/vector.py
@@ -24,6 +24,9 @@
from pathlib import Path
import geopandas as gpd
+import numpy as np
+import shapely
+import xarray as xr
def glaciers_in_complex(rgi_c_id: str, rgi_g: gpd.GeoDataFrame) -> list:
@@ -33,17 +36,25 @@ def glaciers_in_complex(rgi_c_id: str, rgi_g: gpd.GeoDataFrame) -> list:
Parameters
----------
rgi_c_id : str
- The complex outline identifier (e.g. ``"RGI2000-v7.0-C-01-09429"``).
+ The complex outline identifier (e.g. ``"RGI2000-v7.0-C-01-09429"`` or
+ an aggregate name like ``"S4F_AK"``).
rgi_g : geopandas.GeoDataFrame
Glacier outline dataframe with an ``rgi_id_c`` column mapping each
- glacier to its parent complex.
+ glacier to its parent complex. May also have an
+ ``rgi_id_c_aggregate`` column with one or more aggregate-complex
+ names per glacier (semicolon-separated).
Returns
-------
list
- List of ``rgi_id`` strings whose ``rgi_id_c`` matches *rgi_c_id*.
+ List of ``rgi_id`` strings whose ``rgi_id_c`` matches ``rgi_c_id``,
+ plus any whose ``rgi_id_c_aggregate`` lists ``rgi_c_id``.
"""
- return rgi_g.loc[rgi_g["rgi_id_c"] == rgi_c_id, "rgi_id"].tolist()
+ mask = rgi_g["rgi_id_c"] == rgi_c_id
+ if "rgi_id_c_aggregate" in rgi_g.columns:
+ agg = rgi_g["rgi_id_c_aggregate"].fillna("")
+ mask = mask | agg.str.split(";").apply(lambda parts: rgi_c_id in parts)
+ return rgi_g.loc[mask, "rgi_id"].tolist()
def get_glacier_from_rgi_id(rgi: gpd.GeoDataFrame | str | Path, rgi_id: str) -> gpd.GeoDataFrame:
@@ -115,7 +126,56 @@ def aggregate(n, df):
if n == 0:
return df.iloc[[n]]
else:
- geom = df.iloc[range(n)].unary_union
+ geom = df.iloc[range(n)].union_all()
merged_df = df.iloc[[n]]
merged_df.iloc[0].geometry = geom
return merged_df
+
+
+def grid_points_from_dataset(ds: xr.Dataset) -> gpd.GeoDataFrame:
+ """
+ Build a GeoDataFrame of grid cell centers from a domain dataset.
+
+ Parameters
+ ----------
+ ds : xarray.Dataset
+ Dataset with ``x`` and ``y`` coordinates (cell centers) and a
+ ``spatial_ref`` variable carrying CRS information in ``crs_wkt``.
+
+ Returns
+ -------
+ geopandas.GeoDataFrame
+ One Point geometry per cell center, in the dataset's CRS.
+ """
+ xs, ys = np.meshgrid(ds.x.values, ds.y.values)
+ points = shapely.points(xs.ravel(), ys.ravel())
+ return gpd.GeoDataFrame(geometry=points, crs=ds.spatial_ref.attrs.get("crs_wkt"))
+
+
+def grid_cells_from_dataset(ds: xr.Dataset) -> gpd.GeoDataFrame:
+ """
+ Build a GeoDataFrame of grid cell polygons from a domain dataset.
+
+ Parameters
+ ----------
+ ds : xarray.Dataset
+ Dataset with ``x_bnds`` and ``y_bnds`` cell-edge bounds, a ``domain``
+ variable, and a ``spatial_ref`` variable carrying CRS information in
+ ``crs_wkt``.
+
+ Returns
+ -------
+ geopandas.GeoDataFrame
+ One rectangular Polygon per grid cell, with a ``domain`` column
+ copied from ``ds.domain`` and the dataset's CRS.
+ """
+ x0, x1 = ds.x_bnds.values[:, 0], ds.x_bnds.values[:, 1]
+ y0, y1 = ds.y_bnds.values[:, 0], ds.y_bnds.values[:, 1]
+ X0, Y0 = np.meshgrid(x0, y0)
+ X1, Y1 = np.meshgrid(x1, y1)
+ polys = shapely.box(X0.ravel(), Y0.ravel(), X1.ravel(), Y1.ravel())
+ return gpd.GeoDataFrame(
+ {"domain": ds.domain.values.flatten()},
+ geometry=polys,
+ crs=ds.spatial_ref.attrs.get("crs_wkt"),
+ )
diff --git a/pism_terra/workflow.py b/pism_terra/workflow.py
index aafdbd2..2fbb4a7 100644
--- a/pism_terra/workflow.py
+++ b/pism_terra/workflow.py
@@ -24,6 +24,8 @@
from __future__ import annotations
import contextlib
+import logging
+import re
from pathlib import Path
from typing import Any, Iterable, TypeVar
@@ -36,9 +38,65 @@
from pydantic import BaseModel
from tqdm.auto import tqdm
+logger = logging.getLogger(__name__)
+
T = TypeVar("T", bound=BaseModel)
+def parse_cdl_options(cdl_file: str | Path) -> set[str]:
+ """
+ Parse a PISM CDL file and return the set of valid configuration parameter names.
+
+ Extracts names from lines matching ``pism_config: = ...``, ignoring
+ metadata suffixes (``_doc``, ``_type``, ``_units``, ``_option``, ``_choices``,
+ ``_valid_min``, ``_valid_max``).
+
+ Parameters
+ ----------
+ cdl_file : str or Path
+ Path to the CDL file (e.g., ``pism_config.cdl``).
+
+ Returns
+ -------
+ set of str
+ Valid PISM configuration parameter names.
+ """
+ pattern = re.compile(r"^\s*pism_config:(\S+)\s*=")
+ suffixes = ("_doc", "_type", "_units", "_option", "_choices", "_valid_min", "_valid_max")
+ options: set[str] = set()
+ with open(cdl_file, encoding="utf-8") as fh:
+ for line in fh:
+ m = pattern.match(line)
+ if m:
+ name = m.group(1).rstrip(";")
+ if not any(name.endswith(s) for s in suffixes):
+ options.add(name)
+ return options
+
+
+def validate_pism_options(run: dict[str, Any], cdl_file: str | Path) -> None:
+ """
+ Validate that all keys in a PISM run dictionary are recognized config parameters.
+
+ Prints a warning for each key not found in the master CDL file.
+
+ Parameters
+ ----------
+ run : dict
+ Dictionary of PISM run options (dotted keys like ``"surface.pdd.factor_ice"``).
+ cdl_file : str or Path
+ Path to the PISM CDL master config file.
+ """
+ valid = parse_cdl_options(cdl_file)
+ invalid = sorted(k for k in run if k not in valid)
+ if invalid:
+ logger.warning("%d unrecognized PISM option(s):", len(invalid))
+ for k in invalid:
+ logger.warning(" - %s", k)
+ else:
+ logger.info("All %d PISM options are valid.", len(run))
+
+
def merge_model(base_model: T, **overrides: Any) -> T:
"""
Create a new Pydantic model instance by shallow-merging non-None overrides.
@@ -241,6 +299,51 @@ def dict2str(d: dict) -> str:
return """ \\\n""".join(f" -{k} {v}" for k, v in d.items())
+def filter_overrides_by_config(
+ overrides: dict[str, Any], allowed_keys: Iterable[str]
+) -> tuple[dict[str, Any], list[str]]:
+ """
+ Restrict UQ overrides to keys the selected config exposes.
+
+ Drops any entry in ``overrides`` whose key is not in ``allowed_keys``.
+ Returned alongside the kept dict is a sorted list of dropped keys so the
+ caller can surface what was filtered.
+
+ Parameters
+ ----------
+ overrides : dict[str, Any]
+ Candidate dotted PISM-flag overrides (typically the merged uq.toml
+ sample row plus any hardcoded file-path defaults).
+ allowed_keys : Iterable[str]
+ Keys present in the config-derived run dict. Only the selected model's
+ options block contributes here, so an override for a non-selected model
+ (e.g. ``surface.debm_simple.std_dev.file`` when ``surface.model == "pdd"``)
+ is correctly dropped.
+
+ Returns
+ -------
+ kept : dict[str, Any]
+ ``overrides`` filtered to keys present in ``allowed_keys``.
+ skipped : list[str]
+ Keys from ``overrides`` that were not in ``allowed_keys``, sorted.
+
+ Examples
+ --------
+ >>> overrides = {"surface.force_to_thickness.file": "/tmp/boot.nc",
+ ... "surface.debm_simple.std_dev.file": "/tmp/clim.nc"}
+ >>> allowed = {"surface.force_to_thickness.file", "input.file"}
+ >>> kept, skipped = filter_overrides_by_config(overrides, allowed)
+ >>> kept
+ {'surface.force_to_thickness.file': '/tmp/boot.nc'}
+ >>> skipped
+ ['surface.debm_simple.std_dev.file']
+ """
+ allowed = set(allowed_keys)
+ kept = {k: v for k, v in overrides.items() if k in allowed}
+ skipped = sorted(k for k in overrides if k not in allowed)
+ return kept, skipped
+
+
def apply_choice_mapping(uq_df: pd.DataFrame, df: pd.DataFrame, mapping: dict[str, str]) -> pd.DataFrame:
"""
Replace integer choices in `uq_df` with values from `df` using a per-flag mapping.
@@ -477,7 +580,7 @@ def check_dataset_lazy(
# coord monotonicity
for xd in ("x", "lon"):
- if xd in ds.coords:
+ if xd in ds.dims:
xv = ds[xd].values
if xv.size < 2:
raise ValueError(f"Coordinate '{xd}' has <1 elements")
@@ -485,7 +588,7 @@ def check_dataset_lazy(
raise ValueError(f"Coordinate '{xd}' is not strictly monotonic")
for yd in ("y", "lat"):
- if yd in ds.coords:
+ if yd in ds.dims:
yv = ds[yd].values
if yv.size < 2:
raise ValueError(f"Coordinate '{yd}' has <1 elements")
@@ -496,13 +599,10 @@ def check_dataset_lazy(
if "time" in ds.coords:
_ = xr.decode_cf(ds[["time"]]) # raises if invalid CF time
- # CRS (if rioxarray is available)
+ # CRS (if rioxarray is available) — skip silently on any error
try:
- _crs = getattr(ds.rio, "crs", None)
- if _crs is None:
- raise ValueError("Dataset has no CRS (.rio.crs is None)")
+ _crs = ds.rio.crs
except Exception:
- # If rioxarray not present, skip CRS check
pass
def nbytes_est(da: xr.DataArray) -> int:
@@ -558,8 +658,10 @@ def nbytes_est(da: xr.DataArray) -> int:
# Optional: quick global metadata sanity
if ds.attrs.get("Conventions", "").lower().startswith("cf"):
- # try writing minimal in-memory netcdf header/coords only (super light)
- _ = xr.Dataset(coords={k: ds[k].isel({k: slice(0, 1)}) for k in ds.coords})
+ # Try slicing each *dim* coord. Non-dim coords (e.g. ``crs`` /
+ # ``spatial_ref`` grid-mapping placeholders) are 0-D scalars and can't
+ # be isel'd by their own name — skip them.
+ _ = xr.Dataset(coords={k: ds[k].isel({k: slice(0, 1)}) for k in ds.coords if k in ds[k].dims})
# If we reached here, the dataset is healthy enough for downstream steps.
@@ -589,6 +691,124 @@ def check_dataset_fully(ds: xr.Dataset) -> None:
_ = ds.load()
+def drop_geotransform_attr(ds: xr.Dataset | xr.DataArray) -> xr.Dataset | xr.DataArray:
+ """
+ Drop the ``GeoTransform`` attribute from the grid-mapping variable.
+
+ rioxarray writes a ``GeoTransform`` on ``spatial_ref`` whose ``dy`` follows
+ the in-memory y-axis order. With CF-ordered (ascending) y this means
+ ``dy > 0``, which GDAL prefers over the y coordinate variable and so
+ renders the raster upside-down in QGIS. Dropping the attribute makes GDAL
+ fall back to deriving the transform from the y coordinate (top-down).
+
+ Parameters
+ ----------
+ ds : xarray.Dataset or xarray.DataArray
+ Object whose grid-mapping variable/coordinate is modified in place. A
+ ``DataArray`` carries the grid mapping as the ``spatial_ref`` coordinate.
+
+ Returns
+ -------
+ xarray.Dataset or xarray.DataArray
+ The same object, for chaining.
+ """
+ container = ds.coords if isinstance(ds, xr.DataArray) else ds.variables
+ for name in ("spatial_ref", "crs"):
+ if name in container and "GeoTransform" in ds[name].attrs:
+ del ds[name].attrs["GeoTransform"]
+ return ds
+
+
+def stamp_grid_mapping(ds: xr.Dataset, name: str = "spatial_ref") -> xr.Dataset:
+ """
+ Canonicalize the CF grid mapping and drop stray ``coordinates`` attributes.
+
+ The CRS is advertised by the CF ``grid_mapping`` attribute, which names the
+ grid-mapping variable (``spatial_ref``). PISM, GDAL, and QGIS all follow that
+ attribute to read the projection. rioxarray records it in each variable's
+ *encoding*, but operations such as ``concat``/``fillna`` drop encoding, so it
+ can go missing on write — and consumers then fail to find the CRS.
+
+ This finds the real CRS variable (the one carrying a projection WKT or a CF
+ ``grid_mapping_name``), renames it to ``name``, and drops any other CRS
+ variable. Source datasets often ship their own grid mapping (ITS_LIVE names it
+ ``mapping``) that rides through reprojection; an older pipeline could even leave
+ a dangling ``grid_mapping = "spatial_ref"`` pointing at a variable that is
+ actually named ``mapping``. Keying off the metadata rather than the (possibly
+ wrong) attribute fixes both.
+
+ It then re-asserts ``grid_mapping`` on every spatial data variable and strips
+ every grid-mapping name from any ``coordinates`` attribute: ``coordinates`` is
+ for auxiliary coordinate variables (e.g. 2-D lat/lon), and listing the CRS
+ variable there is a CF misuse that confuses PISM (it treats the CRS variable as
+ a data coordinate rather than the projection).
+
+ Stale *empty* grid mappings are also removed. A leftover scalar coordinate (an
+ old ``mapping`` with no projection metadata) survives metadata-based detection,
+ and because it stays attached as a coordinate xarray keeps re-emitting
+ ``coordinates = "mapping"`` on write. Any scalar (0-D) non-dimension coordinate
+ other than the canonical CRS variable is dropped — genuine auxiliary coordinates
+ (lat/lon) are multi-dimensional, so this only catches grid-mapping debris.
+
+ Parameters
+ ----------
+ ds : xarray.Dataset
+ Dataset carrying a CRS variable written by rioxarray (or an upstream
+ source). Not modified in place; callers must use the returned dataset.
+ name : str, default ``"spatial_ref"``
+ Canonical name for the grid-mapping variable.
+
+ Returns
+ -------
+ xarray.Dataset
+ Dataset with a single CRS variable named ``name``, ``grid_mapping``
+ re-asserted on every spatial variable, and no CRS name left in any
+ ``coordinates`` attribute.
+ """
+ # Every variable that looks like a CF grid mapping (carries a projection WKT or
+ # a CF ``grid_mapping_name``) — the real CRS carriers, by name or otherwise.
+ gm_vars = {n for n in ds.variables if "crs_wkt" in ds[n].attrs or "grid_mapping_name" in ds[n].attrs}
+ if not gm_vars:
+ return ds
+
+ # Pick the source CRS variable: the canonical name if it already carries the
+ # metadata, otherwise any real one (e.g. ITS_LIVE's ``mapping``).
+ src = name if name in gm_vars else sorted(gm_vars)[0]
+ if src != name:
+ if name in ds.variables: # a dangling/empty target by the canonical name
+ ds = ds.drop_vars(name)
+ ds = ds.rename({src: name})
+
+ # Drop other real CRS variables and stale empty scalar grid-mapping coordinates.
+ stale = {s for s in gm_vars if s != name}
+ stale |= {c for c in ds.coords if c != name and c not in ds.dims and ds[c].ndim == 0}
+ stale &= set(ds.variables)
+ if stale:
+ ds = ds.drop_vars(list(stale))
+ ds = ds.set_coords(name)
+
+ # Names that must never appear in a ``coordinates`` attribute.
+ crs_names = gm_vars | stale | {name}
+ for var in list(ds.data_vars) + list(ds.coords):
+ if var == name:
+ continue
+ if var in ds.data_vars:
+ ds[var].encoding["grid_mapping"] = name
+ ds[var].attrs.pop("grid_mapping", None) # keep it in encoding only
+ # Strip every grid-mapping name from ``coordinates``, keeping legitimate
+ # auxiliary coordinates if present.
+ for store in (ds[var].attrs, ds[var].encoding):
+ coords = store.get("coordinates")
+ if not coords:
+ continue
+ kept = [c for c in coords.split() if c not in crs_names]
+ if kept:
+ store["coordinates"] = " ".join(kept)
+ else:
+ store.pop("coordinates", None)
+ return ds
+
+
def check_xr_lazy(path: Path | str, verbose: bool = True) -> bool:
"""
Open a dataset and run a **sampled** health check with xarray.
@@ -627,8 +847,10 @@ def check_xr_lazy(path: Path | str, verbose: bool = True) -> bool:
"""
p = Path(path).resolve()
is_ok: bool
+ time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
+ delta_coder = xr.coders.CFTimedeltaCoder()
try:
- ds = xr.open_dataset(p)
+ ds = xr.open_dataset(p, decode_times=time_coder, decode_timedelta=delta_coder)
check_dataset_lazy(ds) # your sampled checker
if verbose:
print(f"{p} is valid ✓")
@@ -670,14 +892,16 @@ def check_xr_fully(path: Path | str) -> bool:
Notes
-----
- Prefer :func:`check_xr_sampled` for very large datasets to avoid
+ Prefer :func:`check_xr_lazy` for very large datasets to avoid
out-of-memory errors. This variant is useful when the dataset is expected
to fit comfortably in memory and you want a strong integrity check.
"""
p = Path(path).resolve()
is_ok: bool
+ time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
+ delta_coder = xr.coders.CFTimedeltaCoder()
try:
- ds = xr.open_dataset(p)
+ ds = xr.open_dataset(p, decode_times=time_coder, decode_timedelta=delta_coder)
check_dataset_fully(ds) # your full-load checker
print(f"{p} is valid ✓")
is_ok = True
diff --git a/pyproject.toml b/pyproject.toml
index ba96a37..ee22510 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,18 +26,21 @@ dependencies = [
"boto3",
"botocore[crt]",
"cdo",
- "cdsapi",
- # "cf-units", # FIXME: Breaks the conda environment; doesn't appear to be needed
"cf_xarray",
+ "cfgrib",
"cftime",
+ "cmap",
+ "cmweather",
"dask[distributed,dataframe]",
"dem_stitcher",
"earthaccess",
+ "ecmwf-datastores-client",
"geocube",
"geopandas",
"h5netcdf",
"jinja2",
"matplotlib",
+ "nc-time-axis",
"netCDf4",
"numpy",
"pandas",
@@ -49,6 +52,7 @@ dependencies = [
"pyogrio",
"pyproj",
"pytest",
+ "pytest-codeblocks>=0.18", # 0.17.x uses the pytest_collect_file(path=...) hook removed in pytest 9.1
"rasterio",
"requests",
"rioxarray",
@@ -74,41 +78,53 @@ dev = [
]
docs = [
- "furo",
+ "ipykernel",
"sphinx",
+ "pydata-sphinx-theme",
+ "sphinx-book-theme",
+ "sphinx-design",
+ "sphinx-gallery",
+ "sphinx-autobuild",
"sphinx-autodoc-typehints",
"sphinx-copybutton",
- "myst-parser",
+ "sphinxcontrib-bibtex",
+ "myst-nb",
"numpydoc",
]
[project.scripts]
-pism-glacier-stage = "pism_terra.glacier.stage:main"
-pism-glacier-run = "pism_terra.glacier.run:run_single"
-pism-glacier-run-ensemble = "pism_terra.glacier.run:run_ensemble"
+combine-crameri-colormaps = "pism_terra.tools.combine_crameri_colormaps:main"
pism-glacier-execute = "pism_terra.glacier.execute:main"
-pism-glacier-prepare = "pism_terra.glacier.prepare:main"
pism-glacier-postprocess = "pism_terra.glacier.postprocess:main"
-pism-ismip7-greenland-prepare = "pism_terra.ismip7.greenland.prepare:main"
+pism-glacier-prepare = "pism_terra.glacier.prepare:rgi"
+pism-glacier-run-inverse = "pism_terra.glacier.run:run_inverse"
+pism-glacier-run-forward = "pism_terra.glacier.run:run_forward"
+pism-glacier-stage = "pism_terra.glacier.stage:main"
+pism-ismip7-greenland-add-basins = "pism_terra.ismip7.greenland.prepare:add_basins"
+pism-ismip7-greenland-postprocess = "pism_terra.ismip7.greenland.postprocess:main"
+pism-ismip7-greenland-prepare = "pism_terra.ismip7.greenland.prepare:cli"
+pism-ismip7-greenland-run-forward = "pism_terra.ismip7.greenland.run:run_forward"
+pism-ismip7-greenland-run-inverse = "pism_terra.ismip7.greenland.run:run_inverse"
pism-ismip7-greenland-stage = "pism_terra.ismip7.greenland.stage:main"
-pism-ismip7-greenland-run = "pism_terra.ismip7.greenland.run:run_single"
-pism-ismip7-greenland-run-ensemble = "pism_terra.ismip7.greenland.run:run_ensemble"
-pism-kitp-prepare = "pism_terra.kitp.prepare:cli"
pism-kitp-postprocess = "pism_terra.kitp.postprocess:main"
-pism-kitp-stage = "pism_terra.kitp.stage:main"
+pism-kitp-prepare = "pism_terra.kitp.prepare:cli"
pism-kitp-run = "pism_terra.kitp.run:run_single"
pism-kitp-run-ensemble = "pism_terra.kitp.run:run_ensemble"
-combine-crameri-colormaps = "pism_terra.tools.combine_crameri_colormaps:main"
+pism-kitp-stage = "pism_terra.kitp.stage:main"
+pism-kitp-writer = "pism_terra.kitp.writer:cli"
+pism-glacier-mip4 = "pism_terra.glacier.glaciermip4:cli"
+pism-s4f-planning = "pism_terra.glacier.s4f:main"
+pism-s4f-prepare = "pism_terra.glacier.prepare:s4f"
+pism-validate = "pism_terra.validate:main"
[tool.setuptools.packages.find]
include = ["pism_terra*"]
-
[tool.setuptools]
py-modules = ["pism_terra"]
[tool.setuptools.package-data]
-"pism_terra" = ['logging.conf', 'data/*.txt', 'data/*.gpkg', 'data/*.toml']
+"pism_terra" = ['logging.conf', 'data/*.txt', 'data/*.gpkg', 'data/*.toml', 'data/*.sh', 'grids/*.txt']
[tool.setuptools_scm]
@@ -196,8 +212,15 @@ disable = """
max-line-length = 120
[tool.pytest.ini_options]
-testpaths = ["tests"]
-markers = "integration"
+markers = [
+ "integration: opt-in slow / network-using tests (run with `pytest -m integration`)",
+]
+testpaths = ["tests", "docs/source"]
+# Markdown fenced blocks are picked up by tests/conftest.py only when the
+# block is explicitly preceded by a ```` comment.
+# Untagged blocks are skipped, so the bulk of the docs aren't accidentally
+# executed.
+addopts = "-ra"
[tool.numpydoc_validation]
checks = [
diff --git a/tests/glacier/test_run.py b/tests/glacier/test_run.py
new file mode 100644
index 0000000..3a696c5
--- /dev/null
+++ b/tests/glacier/test_run.py
@@ -0,0 +1,47 @@
+# Copyright (C) 2025 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+"""
+Test glacier.run functions.
+"""
+
+import pytest
+
+from pism_terra.glacier.run import _nullable_string
+
+
+@pytest.mark.parametrize(
+ "argument_string,expected",
+ [
+ ("None", None),
+ ("none", None),
+ (" NONE ", None),
+ ("foobar", "foobar"),
+ ],
+)
+def test_nullable_string(argument_string, expected) -> None:
+ """
+ Pytest for the glacier.run._nullable_string function.
+
+ Parameters
+ ----------
+ argument_string : str
+ String to be tested.
+ expected : str | None
+ Expected value to be returned.
+ """
+ assert _nullable_string(argument_string) == expected
diff --git a/tests/test_data_sources.py b/tests/test_data_sources.py
new file mode 100644
index 0000000..5f801a5
--- /dev/null
+++ b/tests/test_data_sources.py
@@ -0,0 +1,21 @@
+# Copyright (C) 2025 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+"""
+Test sources.
+"""
diff --git a/tests/test_dem.py b/tests/test_dem.py
index 0ac5d5f..2c9ef54 100644
--- a/tests/test_dem.py
+++ b/tests/test_dem.py
@@ -21,69 +21,15 @@
import geopandas as gpd
import numpy as np
-import pytest
import rasterio
-import xarray as xr
from numpy.testing import assert_array_almost_equal
from rasterio.io import MemoryFile
from shapely.geometry import box
-from pism_terra.glacier.dem import (
- boot_file_from_rgi_id,
+from pism_terra.raster import raster_overlaps_glacier
+from pism_terra.vector import (
get_glacier_from_rgi_id,
)
-from pism_terra.raster import raster_overlaps_glacier
-
-
-@pytest.mark.integration
-def test_boot_file_from_rgi_id(rgi, tmp_path):
- """
- Test ``boot_file_from_rgi_id`` for successful boot dataset creation.
-
- This test verifies that a glacier "boot" dataset (surface, thickness, bed)
- can be generated for a given RGI ID and RGI dataset. It checks that:
-
- * the returned object is an :class:`xarray.Dataset`,
- * expected core variables (``surface``, ``thickness``, ``bed``) are present, and
- * the horizontal spacing of the x-coordinate matches the requested resolution.
-
- Together, this implicitly exercises DEM stitching, reprojection, thickness
- interpolation, and grid construction.
-
- Parameters
- ----------
- rgi : geopandas.GeoDataFrame
- Pre-loaded RGI dataset (typically provided by a pytest fixture) that
- contains a feature with the test ``rgi_id``.
- tmp_path : pathlib.Path
- Pytest-provided temporary directory used to store intermediate and
- output files during the test run.
-
- Notes
- -----
- - This test is intended to run within a pytest test suite.
- - The hard-coded RGI ID (``"RGI2000-v7.0-C-01-10853"``) must be present in
- the ``rgi`` fixture for the test to be meaningful.
- """
- path = tmp_path / "test_boot_file_from_rgi_id"
- path.mkdir(parents=True, exist_ok=True)
-
- rgi_id = "RGI2000-v7.0-C-01-10853"
- resolution = 100.0
- ds = boot_file_from_rgi_id(
- rgi_id,
- rgi,
- dem_dataset="glo_30",
- ice_thickness_dataset="millan",
- velocity_dataset="none",
- resolution=resolution,
- path=path,
- )
- assert isinstance(ds, xr.Dataset)
- assert "surface" in ds
- assert "thickness" in ds
- assert "bed" in ds
- assert abs(ds.x[0] - ds.x[1]) == resolution
def test_get_glacier_from_rgi_id(rgi: gpd.GeoDataFrame):
diff --git a/tests/test_ismip7_counter.py b/tests/test_ismip7_counter.py
new file mode 100644
index 0000000..f6c8115
--- /dev/null
+++ b/tests/test_ismip7_counter.py
@@ -0,0 +1,139 @@
+# Copyright (C) 2026 Andy Aschwanden
+#
+# This file is part of pism-terra.
+#
+# PISM-TERRA is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# PISM-TERRA is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with PISM; if not, write to the Free Software
+
+"""
+Tests for the ISMIP7 Core Experiment counter → config expansion.
+
+Covers:
+- The ``CORE_EXPERIMENTS`` mapping / ``resolve_counter`` helper.
+- ``PismConfig`` expanding ``run_info.counter`` into experiment/pathway/gcms/time.end
+ for every shipped ``ismip7_greenland_c00N.toml`` config.
+- The staged-forcing filename derived from the expanded fields.
+- ``ISMIP7Names`` producing the counter-driven submission path/name.
+- ``time.end`` being required only when no counter is set.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from pism_terra.config import PismConfig, load_config
+from pism_terra.ismip7.experiments import CORE_EXPERIMENTS, resolve_counter
+from pism_terra.ismip7.naming import ISMIP7Names
+
+CONFIG_DIR = Path(__file__).resolve().parents[1] / "pism_terra" / "config"
+
+ALL_COUNTERS = [f"C{n:03d}" for n in range(1, 9)]
+
+
+@pytest.mark.parametrize("counter", ALL_COUNTERS)
+def test_config_expands_counter(counter):
+ """
+ Check each shipped counter config expands to the mapping's derived fields.
+
+ Parameters
+ ----------
+ counter : str
+ ISMIP7 Core Experiment counter id under test (e.g. ``"C003"``).
+ """
+ spec = CORE_EXPERIMENTS[counter]
+ cfg = load_config(CONFIG_DIR / f"ismip7_greenland_{counter.lower()}.toml")
+
+ assert cfg.run_info.counter == counter
+ assert cfg.run_info.experiment == spec.experiment_id
+ assert cfg.campaign.pathway == spec.pathway
+ assert cfg.campaign.gcms == [spec.esm_id]
+ assert cfg.time.time_end == f"{spec.proj_end_year}-01-01"
+
+
+@pytest.mark.parametrize("counter", ALL_COUNTERS)
+def test_forcing_filename_from_expanded_fields(counter):
+ """
+ Check the expanded pathway/gcm/version reproduce the staged forcing filenames.
+
+ Parameters
+ ----------
+ counter : str
+ ISMIP7 Core Experiment counter id under test (e.g. ``"C003"``).
+ """
+ spec = CORE_EXPERIMENTS[counter]
+ cfg = load_config(CONFIG_DIR / f"ismip7_greenland_{counter.lower()}.toml")
+ assert isinstance(cfg.campaign.gcms, list)
+ gcm = cfg.campaign.gcms[0]
+ expected = f"ismip7_greenland_climate_{spec.pathway}_{spec.esm_id}_{cfg.campaign.version}.nc"
+ assert f"ismip7_greenland_climate_{cfg.campaign.pathway}_{gcm}_{cfg.campaign.version}.nc" == expected
+
+
+def test_resolve_counter_unknown_raises():
+ """An unsupported counter id raises a helpful ValueError."""
+ with pytest.raises(ValueError, match="unknown ISMIP7 counter"):
+ resolve_counter("C099")
+
+
+def test_resolve_counter_case_insensitive():
+ """Counter lookup is case-insensitive and trims whitespace."""
+ assert resolve_counter(" c003 ") is CORE_EXPERIMENTS["C003"]
+
+
+def test_historical_counters_use_ssp585_to_2100():
+ """C001/C002 keep the historical product but continue on ssp585 to 2100."""
+ for counter in ("C001", "C002"):
+ spec = CORE_EXPERIMENTS[counter]
+ assert spec.experiment_id == "historical"
+ assert spec.product_leg == "historical"
+ assert spec.pathway == "ssp585"
+ assert spec.proj_end_year == 2100
+
+
+def test_ismip7_names_use_counter():
+ """Build the counter-driven ISMIP7 submission directory and file name."""
+ names = ISMIP7Names(
+ domain_id="GrIS",
+ source_id="UAF",
+ ism_id="PISM",
+ ism_member_id="m001",
+ esm_id="CESM2-WACCM",
+ forcing_member_id="f001",
+ experiment_id="ssp370",
+ set_id="CORE",
+ set_counter="C003",
+ time_range="2015-2100",
+ )
+ assert names.directory("/root").as_posix().endswith("GrIS/UAF/PISM/CORE/C003")
+ assert "_ssp370_C003_" in names.filename("acabf")
+
+
+def test_time_end_required_without_counter():
+ """A config with neither counter nor time.end fails validation."""
+ data = {
+ "run_info": {"run_info.title": "no end"},
+ "campaign": {},
+ "time": {"time.start": "1985-01-01"},
+ "grid": {"resolution": "900m"},
+ "energy": {"model": "enthalpy", "options": {"enthalpy": {}}},
+ "stress_balance": {"model": "blatter", "options": {"blatter": {}}},
+ "atmosphere": {"model": "given", "options": {"given": {}}},
+ "ocean": {"model": "th", "options": {"th": {}}},
+ "surface": {"model": "given", "options": {"given": {}}},
+ "frontal_melt": {"model": "routing", "options": {"routing": {}}},
+ "bed_deformation": {"model": "none", "options": {"none": {}}},
+ "hydrology": {"model": "routing", "options": {"routing": {}}},
+ }
+ with pytest.raises(ValueError, match="time.end is required"):
+ PismConfig.model_validate(data)