From 49d0de259de1d2d8c8db9445a76ba618d3869292 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Sun, 5 Jul 2026 11:14:46 -0400 Subject: [PATCH 01/26] make actions more robust to network failures --- .github/actions/download-demo/action.yml | 8 ++++++-- .github/actions/setup-duckdb/action.yml | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/actions/download-demo/action.yml b/.github/actions/download-demo/action.yml index d4a1a70c..698df4ce 100644 --- a/.github/actions/download-demo/action.yml +++ b/.github/actions/download-demo/action.yml @@ -27,6 +27,10 @@ runs: shell: bash working-directory: ${{ inputs.dest }} run: | + set -euo pipefail echo "Downloading MIMIC-IV and MIMIC-IV-ED demo from PhysioNet." - wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/ - wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/ + # -4 forces IPv4 + # -c resumes any partially downloaded files on retry. + wget_opts="-4 --tries=5 --retry-connrefused --waitretry=10 --timeout=60" + wget -r -N -c -q $wget_opts --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/ + wget -r -N -c -q $wget_opts --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/ diff --git a/.github/actions/setup-duckdb/action.yml b/.github/actions/setup-duckdb/action.yml index c358e449..91ba9350 100644 --- a/.github/actions/setup-duckdb/action.yml +++ b/.github/actions/setup-duckdb/action.yml @@ -13,7 +13,11 @@ runs: - name: Install DuckDB CLI shell: bash run: | - wget -q "https://github.com/duckdb/duckdb/releases/download/v${{ inputs.version }}/duckdb_cli-linux-amd64.zip" + set -euo pipefail + url="https://github.com/duckdb/duckdb/releases/download/v${{ inputs.version }}/duckdb_cli-linux-amd64.zip" + # -4 forces IPv4, plus retries to survive transient network failures + wget -q -4 --tries=5 --retry-connrefused --waitretry=10 --timeout=60 \ + -O duckdb_cli-linux-amd64.zip "$url" unzip -o duckdb_cli-linux-amd64.zip -d /usr/local/bin rm duckdb_cli-linux-amd64.zip duckdb --version From 239c3496ac7e29f9db6d66f3317ed740985402aa Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Sun, 5 Jul 2026 11:15:32 -0400 Subject: [PATCH 02/26] temporarily add triggering of ci on push to branch --- .github/workflows/ci.yml | 10 ++++++++-- .github/workflows/mysql.yml | 15 +++++++++++---- .github/workflows/sqlite.yml | 7 ++++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31b34c31..70eee0f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,14 +13,20 @@ on: pull_request_review: types: [submitted] workflow_dispatch: + # TEMP: test on push to branch + push: + branches: + - alistair/more_gh_action_shenanigans jobs: psql: - if: github.event_name == 'workflow_dispatch' || github.event.review.state == 'approved' + # TEMP: 'push' added to run on branch push + if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || github.event.review.state == 'approved' uses: ./.github/workflows/build-psql.yml duckdb: - if: github.event_name == 'workflow_dispatch' || github.event.review.state == 'approved' + # TEMP: 'push' added to run on branch push + if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || github.event.review.state == 'approved' uses: ./.github/workflows/build-duckdb.yml psql-concepts: diff --git a/.github/workflows/mysql.yml b/.github/workflows/mysql.yml index 2f1bc695..73b56cc8 100644 --- a/.github/workflows/mysql.yml +++ b/.github/workflows/mysql.yml @@ -2,11 +2,16 @@ name: mysql demo db build on: pull_request_review: types: [submitted] + # TEMP: test on push to branch + push: + branches: + - alistair/more_gh_action_shenanigans jobs: mimic-iv-mysql: # only run if PR is approved - if: github.event.review.state == 'approved' + # TEMP: also run on push + if: github.event_name == 'push' || github.event.review.state == 'approved' runs-on: ubuntu-22.04 steps: @@ -18,9 +23,11 @@ jobs: - name: Extract demo data to local folder run: | - mv hosp/*.csv.gz ./ - mv icu/*.csv.gz ./ - mv ed/*.csv.gz ./ + # download-demo provides the gzipped CSVs under hosp/, icu/ and ed/. + # Flatten every nested CSV into the working directory, where the mysql + # load.sql scripts expect them (e.g. LOAD DATA INFILE 'admissions.csv'). + # Using find keeps this robust to how deeply the files are nested. + find . -mindepth 2 -name '*.csv.gz' -exec mv -t ./ {} + gzip -d *.csv.gz - name: Start MySQL service diff --git a/.github/workflows/sqlite.yml b/.github/workflows/sqlite.yml index e4867604..d65a66ee 100644 --- a/.github/workflows/sqlite.yml +++ b/.github/workflows/sqlite.yml @@ -2,11 +2,16 @@ name: sqlite demo db build on: pull_request_review: types: [submitted] + # TEMP: test on push to branch + push: + branches: + - alistair/more_gh_action_shenanigans jobs: mimic-iv-sqlite: # only run if PR is approved - if: github.event.review.state == 'approved' + # TEMP: also run on push + if: github.event_name == 'push' || github.event.review.state == 'approved' runs-on: ubuntu-latest container: python:3.10 From 70bc25c0695ee98ee6ae41b2bad5f7a4dda5e8d8 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Sun, 5 Jul 2026 11:45:50 -0400 Subject: [PATCH 03/26] fix incorrect transpilation to tz types, add tests --- src/mimic_utils/compare_concepts.py | 15 +++++++++++++ src/mimic_utils/sqlglot_dialects/duckdb.py | 2 ++ src/mimic_utils/sqlglot_dialects/postgres.py | 5 +++++ src/mimic_utils/transpile.py | 10 ++++++++- tests/test_compare_concepts.py | 22 ++++++++++++++++++++ tests/test_transpile.py | 12 +++++++++++ 6 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 tests/test_compare_concepts.py diff --git a/src/mimic_utils/compare_concepts.py b/src/mimic_utils/compare_concepts.py index c0e03412..2c345e9d 100644 --- a/src/mimic_utils/compare_concepts.py +++ b/src/mimic_utils/compare_concepts.py @@ -55,12 +55,25 @@ def _list_tables_duckdb(con, schema): def _as_numeric(series): """Return a float Series if every non-null value coerces, else None.""" + if _is_datetime_like_series(series): + return None coerced = pd.to_numeric(series, errors="coerce") if coerced.isna().sum() == series.isna().sum(): return coerced.astype(float) return None +def _is_datetime_like_series(series): + """Return True when values are already datetimes, even if stored as object dtype.""" + if pd.api.types.is_datetime64_any_dtype(series.dtype) or pd.api.types.is_datetime64tz_dtype(series.dtype): + return True + non_null = series.dropna() + if non_null.empty: + return False + sample = non_null.iloc[0] + return isinstance(sample, (pd.Timestamp, np.datetime64)) or hasattr(sample, "tzinfo") + + def _as_datetime(series): """Return a datetime Series if every non-null value parses, else None.""" import warnings @@ -68,6 +81,8 @@ def _as_datetime(series): warnings.simplefilter("ignore") # silence per-element parse fallback notices coerced = pd.to_datetime(series, errors="coerce") if coerced.isna().sum() == series.isna().sum(): + if pd.api.types.is_datetime64tz_dtype(coerced.dtype): + coerced = coerced.dt.tz_localize(None) return coerced return None diff --git a/src/mimic_utils/sqlglot_dialects/duckdb.py b/src/mimic_utils/sqlglot_dialects/duckdb.py index 48e5e135..9d282ad2 100644 --- a/src/mimic_utils/sqlglot_dialects/duckdb.py +++ b/src/mimic_utils/sqlglot_dialects/duckdb.py @@ -11,6 +11,8 @@ class MimicDuckDB(DuckDB): class Generator(DuckDB.Generator): def datatype_sql(self, expression: exp.DataType) -> str: + if expression.this == exp.DataType.Type.TIMESTAMPTZ: + return "TIMESTAMP" if expression.this == exp.DataType.Type.DECIMAL and not expression.expressions: return "DECIMAL(38, 9)" return super().datatype_sql(expression) diff --git a/src/mimic_utils/sqlglot_dialects/postgres.py b/src/mimic_utils/sqlglot_dialects/postgres.py index 1048fe8c..af6a1fe3 100644 --- a/src/mimic_utils/sqlglot_dialects/postgres.py +++ b/src/mimic_utils/sqlglot_dialects/postgres.py @@ -80,6 +80,11 @@ def _generate_array_sql(self: Postgres.Generator, expression: exp.Expression) -> class MimicPostgres(Postgres): class Generator(Postgres.Generator): + def datatype_sql(self, expression: exp.DataType) -> str: + if expression.this == exp.DataType.Type.TIMESTAMPTZ: + return "TIMESTAMP" + return super().datatype_sql(expression) + TRANSFORMS = { **Postgres.Generator.TRANSFORMS, exp.DatetimeDiff: _datetime_diff_sql, diff --git a/src/mimic_utils/transpile.py b/src/mimic_utils/transpile.py index 8487456b..51403940 100644 --- a/src/mimic_utils/transpile.py +++ b/src/mimic_utils/transpile.py @@ -1,5 +1,6 @@ import logging import os +import re from pathlib import Path from typing import Union @@ -25,6 +26,12 @@ _CATALOG_TO_REMOVE = "physionet-data" +def _strip_timezone_types(sql: str) -> str: + """Normalize any timezone-aware timestamp types back to naive TIMESTAMP.""" + sql = re.sub(r"\bTIMESTAMP\s+WITH\s+TIME\s+ZONE\b", "TIMESTAMP", sql) + return re.sub(r"\bTIMESTAMPTZ\b", "TIMESTAMP", sql) + + def _strip_catalog(parsed: exp.Expression) -> None: """Remove the ``physionet-data`` catalog from every table reference in place.""" for table in parsed.find_all(exp.Table): @@ -50,11 +57,12 @@ def transpile_query(query: str, source_dialect: str = "bigquery", destination_di # drop comments when targeting it. keep_comments = destination_dialect != "duckdb" - return parsed.sql( + sql = parsed.sql( dialect=_DESTINATION_DIALECTS[destination_dialect], pretty=True, comments=keep_comments, ) + return _strip_timezone_types(sql) def transpile_file( diff --git a/tests/test_compare_concepts.py b/tests/test_compare_concepts.py new file mode 100644 index 00000000..5ad98e28 --- /dev/null +++ b/tests/test_compare_concepts.py @@ -0,0 +1,22 @@ +from datetime import datetime, timedelta, timezone + +import pandas as pd + +from mimic_utils.compare_concepts import compare_table + + +def test_compare_table_treats_same_wall_time_as_equal_across_timezones(): + pg = pd.DataFrame({"charttime": [datetime(2150, 1, 1, 3, 30, tzinfo=timezone(timedelta(hours=2)))]}) + duck = pd.DataFrame({"charttime": [datetime(2150, 1, 1, 3, 30)]}) + + assert compare_table(pg, duck, 1e-6, 1e-9) == (True, "1 rows OK") + + +def test_compare_table_flags_different_wall_times_even_if_instants_match(): + pg = pd.DataFrame({"charttime": [datetime(2150, 1, 1, 3, 30, tzinfo=timezone(timedelta(hours=2)))]}) + duck = pd.DataFrame({"charttime": [datetime(2150, 1, 1, 1, 30)]}) + + assert compare_table(pg, duck, 1e-6, 1e-9) == ( + False, + "1 differing cells in columns: charttime(1)", + ) \ No newline at end of file diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 67470da5..14832048 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -102,6 +102,18 @@ def test_duckdb_comments_stripped(): assert "/*" in transpile_query(bq, "bigquery", "postgres") +@pytest.mark.parametrize( + "bq", + [ + "SELECT TIMESTAMP('2150-01-01 00:00:00') AS ts", + "SELECT CAST(DATETIME '2150-01-01 00:00:00' AS TIMESTAMP) AS ts", + ], +) +def test_timestamp_outputs_stay_timezone_naive(bq): + for dialect in ("postgres", "duckdb"): + assert "TIMESTAMPTZ" not in transpile_query(bq, "bigquery", dialect) + + def test_unsupported_dialect_raises(): with pytest.raises(ValueError): transpile_query("SELECT 1", "bigquery", "mysql") From d3f593d7a0d2e4acefeb59a7ddd83e55925d8707 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Sun, 5 Jul 2026 12:00:03 -0400 Subject: [PATCH 04/26] change string_agg to return deterministic output --- mimic-iv/concepts/firstday/first_day_rrt.sql | 2 +- mimic-iv/concepts_duckdb/firstday/first_day_rrt.sql | 4 +++- mimic-iv/concepts_postgres/firstday/first_day_rrt.sql | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mimic-iv/concepts/firstday/first_day_rrt.sql b/mimic-iv/concepts/firstday/first_day_rrt.sql index 2f99f12e..b6e95a74 100644 --- a/mimic-iv/concepts/firstday/first_day_rrt.sql +++ b/mimic-iv/concepts/firstday/first_day_rrt.sql @@ -5,7 +5,7 @@ SELECT , ie.stay_id , MAX(dialysis_present) AS dialysis_present , MAX(dialysis_active) AS dialysis_active - , STRING_AGG(DISTINCT dialysis_type, ', ') AS dialysis_type + , STRING_AGG(DISTINCT dialysis_type, ', ' ORDER BY dialysis_type) AS dialysis_type FROM `physionet-data.mimiciv_icu.icustays` ie LEFT JOIN `physionet-data.mimiciv_derived.rrt` rrt ON ie.stay_id = rrt.stay_id diff --git a/mimic-iv/concepts_duckdb/firstday/first_day_rrt.sql b/mimic-iv/concepts_duckdb/firstday/first_day_rrt.sql index de2901e0..3c710580 100644 --- a/mimic-iv/concepts_duckdb/firstday/first_day_rrt.sql +++ b/mimic-iv/concepts_duckdb/firstday/first_day_rrt.sql @@ -5,7 +5,9 @@ SELECT ie.stay_id, MAX(dialysis_present) AS dialysis_present, MAX(dialysis_active) AS dialysis_active, - LISTAGG(DISTINCT dialysis_type, ', ') AS dialysis_type + LISTAGG(DISTINCT dialysis_type, ', ' + ORDER BY + dialysis_type NULLS FIRST) AS dialysis_type FROM mimiciv_icu.icustays AS ie LEFT JOIN mimiciv_derived.rrt AS rrt ON ie.stay_id = rrt.stay_id diff --git a/mimic-iv/concepts_postgres/firstday/first_day_rrt.sql b/mimic-iv/concepts_postgres/firstday/first_day_rrt.sql index 1b7b231b..904bcf2a 100644 --- a/mimic-iv/concepts_postgres/firstday/first_day_rrt.sql +++ b/mimic-iv/concepts_postgres/firstday/first_day_rrt.sql @@ -6,7 +6,9 @@ SELECT ie.stay_id, MAX(dialysis_present) AS dialysis_present, MAX(dialysis_active) AS dialysis_active, - STRING_AGG(DISTINCT dialysis_type, ', ') AS dialysis_type + STRING_AGG(DISTINCT dialysis_type, ', ' + ORDER BY + dialysis_type NULLS FIRST) AS dialysis_type FROM mimiciv_icu.icustays AS ie LEFT JOIN mimiciv_derived.rrt AS rrt ON ie.stay_id = rrt.stay_id From 2afa73c0d9249f82e2ef014b2df70cc61cfbbd66 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Sun, 5 Jul 2026 12:00:42 -0400 Subject: [PATCH 05/26] change rhythm values to concatenate multiple rhythms rather than arbitrarily take a MAX() --- mimic-iv/concepts/measurement/rhythm.sql | 5 ++++- mimic-iv/concepts_duckdb/measurement/rhythm.sql | 6 +++++- mimic-iv/concepts_postgres/measurement/rhythm.sql | 6 +++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/mimic-iv/concepts/measurement/rhythm.sql b/mimic-iv/concepts/measurement/rhythm.sql index 20b9d8b5..dcf35836 100644 --- a/mimic-iv/concepts/measurement/rhythm.sql +++ b/mimic-iv/concepts/measurement/rhythm.sql @@ -2,7 +2,10 @@ SELECT ce.subject_id , ce.charttime - , MAX(CASE WHEN itemid = 220048 THEN value ELSE NULL END) AS heart_rhythm + , STRING_AGG( + DISTINCT CASE WHEN itemid = 220048 THEN value ELSE NULL END, + '; ' ORDER BY CASE WHEN itemid = 220048 THEN value ELSE NULL END + ) AS heart_rhythm , MAX(CASE WHEN itemid = 224650 THEN value ELSE NULL END) AS ectopy_type , MAX( CASE WHEN itemid = 224651 THEN value ELSE NULL END diff --git a/mimic-iv/concepts_duckdb/measurement/rhythm.sql b/mimic-iv/concepts_duckdb/measurement/rhythm.sql index 9a570a97..1907a9e2 100644 --- a/mimic-iv/concepts_duckdb/measurement/rhythm.sql +++ b/mimic-iv/concepts_duckdb/measurement/rhythm.sql @@ -3,7 +3,11 @@ DROP TABLE IF EXISTS mimiciv_derived.rhythm; CREATE TABLE mimiciv_derived.rhythm SELECT ce.subject_id, ce.charttime, - MAX(CASE WHEN itemid = 220048 THEN value ELSE NULL END) AS heart_rhythm, + LISTAGG( + DISTINCT CASE WHEN itemid = 220048 THEN value ELSE NULL END, '; ' + ORDER BY + CASE WHEN itemid = 220048 THEN value ELSE NULL END NULLS FIRST + ) AS heart_rhythm, MAX(CASE WHEN itemid = 224650 THEN value ELSE NULL END) AS ectopy_type, MAX(CASE WHEN itemid = 224651 THEN value ELSE NULL END) AS ectopy_frequency, MAX(CASE WHEN itemid = 226479 THEN value ELSE NULL END) AS ectopy_type_secondary, diff --git a/mimic-iv/concepts_postgres/measurement/rhythm.sql b/mimic-iv/concepts_postgres/measurement/rhythm.sql index 243410f6..7eba65d0 100644 --- a/mimic-iv/concepts_postgres/measurement/rhythm.sql +++ b/mimic-iv/concepts_postgres/measurement/rhythm.sql @@ -4,7 +4,11 @@ DROP TABLE IF EXISTS mimiciv_derived.rhythm; CREATE TABLE mimiciv_derived.rhythm SELECT ce.subject_id, ce.charttime, - MAX(CASE WHEN itemid = 220048 THEN value ELSE NULL END) AS heart_rhythm, + STRING_AGG( + DISTINCT CASE WHEN itemid = 220048 THEN value ELSE NULL END, '; ' + ORDER BY + CASE WHEN itemid = 220048 THEN value ELSE NULL END NULLS FIRST + ) AS heart_rhythm, MAX(CASE WHEN itemid = 224650 THEN value ELSE NULL END) AS ectopy_type, MAX(CASE WHEN itemid = 224651 THEN value ELSE NULL END) AS ectopy_frequency, MAX(CASE WHEN itemid = 226479 THEN value ELSE NULL END) AS ectopy_type_secondary, From e68ccb24990767719d61a6b116213a9cf620ffc5 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Sun, 5 Jul 2026 23:11:33 -0400 Subject: [PATCH 06/26] ensure postgres also matches bigquery numeric precision --- src/mimic_utils/sqlglot_dialects/postgres.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mimic_utils/sqlglot_dialects/postgres.py b/src/mimic_utils/sqlglot_dialects/postgres.py index af6a1fe3..02a2f93f 100644 --- a/src/mimic_utils/sqlglot_dialects/postgres.py +++ b/src/mimic_utils/sqlglot_dialects/postgres.py @@ -83,6 +83,8 @@ class Generator(Postgres.Generator): def datatype_sql(self, expression: exp.DataType) -> str: if expression.this == exp.DataType.Type.TIMESTAMPTZ: return "TIMESTAMP" + if expression.this == exp.DataType.Type.DECIMAL and not expression.expressions: + return "DECIMAL(38, 9)" return super().datatype_sql(expression) TRANSFORMS = { From 824b421c2b0f6b437d52317248e1e9064d60ac65 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Sun, 5 Jul 2026 23:11:55 -0400 Subject: [PATCH 07/26] test numeric precision for transpile --- tests/test_transpile.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 14832048..20adcbb5 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -72,12 +72,10 @@ def t(bq: str, dialect: str) -> str: ("int64_cast_pg", "SELECT CAST(hr AS INT64) AS hr FROM t", "postgres", "SELECT CAST(hr AS BIGINT) AS hr FROM t"), - # BigQuery NUMERIC == DECIMAL(38, 9); postgres NUMERIC is arbitrary precision - # (keeps the value), but DuckDB's bare DECIMAL defaults to scale 3 and rounds, - # so we must pin the precision there. - # postgres DECIMAL == NUMERIC, both arbitrary precision, so the value is kept + # BigQuery NUMERIC == DECIMAL(38, 9), so both target dialects must pin the + # precision and scale explicitly to preserve BigQuery semantics. ("numeric_cast_pg", "SELECT CAST(x AS NUMERIC) FROM t", "postgres", - "SELECT CAST(x AS DECIMAL) FROM t"), + "SELECT CAST(x AS DECIMAL(38, 9)) FROM t"), ("numeric_cast_duckdb", "SELECT CAST(x AS NUMERIC) FROM t", "duckdb", "SELECT CAST(x AS DECIMAL(38, 9)) FROM t"), ] From 317068a1f4435b48e45e48027e12c875b8ed27ef Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Sun, 5 Jul 2026 23:12:16 -0400 Subject: [PATCH 08/26] fix rounding errors causing slight mismatches in concepts --- .../demographics/weight_durations.sql | 4 +- mimic-iv/concepts/measurement/bg.sql | 4 +- .../measurement/urine_output_rate.sql | 18 ++++--- mimic-iv/concepts/organfailure/kdigo_uo.sql | 12 ++--- .../demographics/weight_durations.sql | 7 ++- mimic-iv/concepts_duckdb/measurement/bg.sql | 4 +- .../measurement/urine_output_rate.sql | 36 +++++++------ .../concepts_duckdb/organfailure/kdigo_uo.sql | 33 +++++++----- .../demographics/icustay_detail.sql | 2 +- .../demographics/weight_durations.sql | 5 +- .../firstday/first_day_height.sql | 2 +- mimic-iv/concepts_postgres/measurement/bg.sql | 4 +- .../measurement/blood_differential.sql | 10 ++-- .../concepts_postgres/measurement/height.sql | 4 +- .../measurement/urine_output_rate.sql | 51 +++++++++++-------- .../measurement/vitalsign.sql | 2 +- .../norepinephrine_equivalent_dose.sql | 2 +- .../organfailure/kdigo_uo.sql | 39 ++++++++------ .../concepts_postgres/organfailure/meld.sql | 2 +- 19 files changed, 141 insertions(+), 100 deletions(-) diff --git a/mimic-iv/concepts/demographics/weight_durations.sql b/mimic-iv/concepts/demographics/weight_durations.sql index de1b5cdf..a28bb4f6 100644 --- a/mimic-iv/concepts/demographics/weight_durations.sql +++ b/mimic-iv/concepts/demographics/weight_durations.sql @@ -6,8 +6,7 @@ WITH wt_stg AS ( , c.charttime , CASE WHEN c.itemid = 226512 THEN 'admit' ELSE 'daily' END AS weight_type - -- TODO: eliminate obvious outliers if there is a reasonable weight - , c.valuenum AS weight + , ROUND(CAST(c.valuenum AS NUMERIC), 3) AS weight FROM `physionet-data.mimiciv_icu.chartevents` c WHERE c.valuenum IS NOT NULL AND c.itemid IN @@ -16,6 +15,7 @@ WITH wt_stg AS ( , 224639 -- Daily Weight ) AND c.valuenum > 0 + AND c.valuenum < 1500 ) -- assign ascending row number diff --git a/mimic-iv/concepts/measurement/bg.sql b/mimic-iv/concepts/measurement/bg.sql index c464acd4..a5b393ef 100644 --- a/mimic-iv/concepts/measurement/bg.sql +++ b/mimic-iv/concepts/measurement/bg.sql @@ -148,7 +148,7 @@ WITH bg AS ( , stg2 AS ( SELECT bg.* , ROW_NUMBER() OVER ( - PARTITION BY bg.subject_id, bg.charttime ORDER BY s1.charttime DESC + PARTITION BY bg.specimen_id ORDER BY s1.charttime DESC ) AS lastrowspo2 , s1.spo2 FROM bg @@ -165,7 +165,7 @@ WITH bg AS ( , stg3 AS ( SELECT bg.* , ROW_NUMBER() OVER ( - PARTITION BY bg.subject_id, bg.charttime ORDER BY s2.charttime DESC + PARTITION BY bg.specimen_id ORDER BY s2.charttime DESC ) AS lastrowfio2 , s2.fio2_chartevents FROM stg2 bg diff --git a/mimic-iv/concepts/measurement/urine_output_rate.sql b/mimic-iv/concepts/measurement/urine_output_rate.sql index 0a9a2264..99b1311e 100644 --- a/mimic-iv/concepts/measurement/urine_output_rate.sql +++ b/mimic-iv/concepts/measurement/urine_output_rate.sql @@ -48,18 +48,22 @@ WITH tm AS ( , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_6hr - , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 - THEN iosum.tm_since_last_uo - ELSE NULL END) / 60.0 AS uo_tm_6hr + , ROUND(CAST( + SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 + THEN iosum.tm_since_last_uo + ELSE NULL END) / 60.0 AS NUMERIC + ), 6) AS uo_tm_6hr , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_12hr - , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 - THEN iosum.tm_since_last_uo - ELSE NULL END) / 60.0 AS uo_tm_12hr + , ROUND(CAST( + SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 + THEN iosum.tm_since_last_uo + ELSE NULL END) / 60.0 AS NUMERIC + ), 6) AS uo_tm_12hr -- 24 hours , SUM(iosum.urineoutput) AS urineoutput_24hr - , SUM(iosum.tm_since_last_uo) / 60.0 AS uo_tm_24hr + , ROUND(CAST(SUM(iosum.tm_since_last_uo) / 60.0 AS NUMERIC), 6) AS uo_tm_24hr FROM uo_tm io -- this join gives you all UO measurements over a 24 hour period diff --git a/mimic-iv/concepts/organfailure/kdigo_uo.sql b/mimic-iv/concepts/organfailure/kdigo_uo.sql index 32119dc9..c85d4223 100644 --- a/mimic-iv/concepts/organfailure/kdigo_uo.sql +++ b/mimic-iv/concepts/organfailure/kdigo_uo.sql @@ -45,26 +45,26 @@ WITH uo_stg1 AS ( -- repeat the summations using the hours_since_previous_row column -- this gives us the amount of time the UO was calculated over - , SUM(hours_since_previous_row) OVER + , ROUND(CAST(SUM(hours_since_previous_row) OVER ( PARTITION BY stay_id ORDER BY seconds_since_admit RANGE BETWEEN 21600 PRECEDING AND CURRENT ROW - ) AS uo_tm_6hr + ) AS NUMERIC), 6) AS uo_tm_6hr - , SUM(hours_since_previous_row) OVER + , ROUND(CAST(SUM(hours_since_previous_row) OVER ( PARTITION BY stay_id ORDER BY seconds_since_admit RANGE BETWEEN 43200 PRECEDING AND CURRENT ROW - ) AS uo_tm_12hr + ) AS NUMERIC), 6) AS uo_tm_12hr - , SUM(hours_since_previous_row) OVER + , ROUND(CAST(SUM(hours_since_previous_row) OVER ( PARTITION BY stay_id ORDER BY seconds_since_admit RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW - ) AS uo_tm_24hr + ) AS NUMERIC), 6) AS uo_tm_24hr FROM uo_stg1 ) diff --git a/mimic-iv/concepts_duckdb/demographics/weight_durations.sql b/mimic-iv/concepts_duckdb/demographics/weight_durations.sql index b30ea009..f47d63d0 100644 --- a/mimic-iv/concepts_duckdb/demographics/weight_durations.sql +++ b/mimic-iv/concepts_duckdb/demographics/weight_durations.sql @@ -5,10 +5,13 @@ WITH wt_stg AS ( c.stay_id, c.charttime, CASE WHEN c.itemid = 226512 THEN 'admit' ELSE 'daily' END AS weight_type, - c.valuenum AS weight + ROUND(CAST(c.valuenum AS DECIMAL(38, 9)), 3) AS weight FROM mimiciv_icu.chartevents AS c WHERE - NOT c.valuenum IS NULL AND c.itemid IN (226512, 224639) AND c.valuenum > 0 + NOT c.valuenum IS NULL + AND c.itemid IN (226512, 224639) + AND c.valuenum > 0 + AND c.valuenum < 1500 ), wt_stg1 AS ( SELECT stay_id, diff --git a/mimic-iv/concepts_duckdb/measurement/bg.sql b/mimic-iv/concepts_duckdb/measurement/bg.sql index 01e87e5c..9fc631da 100644 --- a/mimic-iv/concepts_duckdb/measurement/bg.sql +++ b/mimic-iv/concepts_duckdb/measurement/bg.sql @@ -110,7 +110,7 @@ WITH bg AS ( ), stg2 AS ( SELECT bg.*, - ROW_NUMBER() OVER (PARTITION BY bg.subject_id, bg.charttime ORDER BY s1.charttime DESC) AS lastrowspo2, + ROW_NUMBER() OVER (PARTITION BY bg.specimen_id ORDER BY s1.charttime DESC) AS lastrowspo2, s1.spo2 FROM bg LEFT JOIN stg_spo2 AS s1 @@ -121,7 +121,7 @@ WITH bg AS ( ), stg3 AS ( SELECT bg.*, - ROW_NUMBER() OVER (PARTITION BY bg.subject_id, bg.charttime ORDER BY s2.charttime DESC) AS lastrowfio2, + ROW_NUMBER() OVER (PARTITION BY bg.specimen_id ORDER BY s2.charttime DESC) AS lastrowfio2, s2.fio2_chartevents FROM stg2 AS bg LEFT JOIN stg_fio2 AS s2 diff --git a/mimic-iv/concepts_duckdb/measurement/urine_output_rate.sql b/mimic-iv/concepts_duckdb/measurement/urine_output_rate.sql index 21f9c491..f39e4b41 100644 --- a/mimic-iv/concepts_duckdb/measurement/urine_output_rate.sql +++ b/mimic-iv/concepts_duckdb/measurement/urine_output_rate.sql @@ -39,13 +39,16 @@ WITH tm AS ( ELSE NULL END ) AS urineoutput_6hr, - SUM( - CASE - WHEN DATE_DIFF('HOUR', iosum.charttime, io.charttime) <= 5 - THEN iosum.tm_since_last_uo - ELSE NULL - END - ) / 60.0 AS uo_tm_6hr, + ROUND( + CAST(SUM( + CASE + WHEN DATE_DIFF('HOUR', iosum.charttime, io.charttime) <= 5 + THEN iosum.tm_since_last_uo + ELSE NULL + END + ) / 60.0 AS DECIMAL(38, 9)), + 6 + ) AS uo_tm_6hr, SUM( CASE WHEN DATE_DIFF('HOUR', iosum.charttime, io.charttime) <= 11 @@ -53,15 +56,18 @@ WITH tm AS ( ELSE NULL END ) AS urineoutput_12hr, - SUM( - CASE - WHEN DATE_DIFF('HOUR', iosum.charttime, io.charttime) <= 11 - THEN iosum.tm_since_last_uo - ELSE NULL - END - ) / 60.0 AS uo_tm_12hr, + ROUND( + CAST(SUM( + CASE + WHEN DATE_DIFF('HOUR', iosum.charttime, io.charttime) <= 11 + THEN iosum.tm_since_last_uo + ELSE NULL + END + ) / 60.0 AS DECIMAL(38, 9)), + 6 + ) AS uo_tm_12hr, SUM(iosum.urineoutput) AS urineoutput_24hr, - SUM(iosum.tm_since_last_uo) / 60.0 AS uo_tm_24hr + ROUND(CAST(SUM(iosum.tm_since_last_uo) / 60.0 AS DECIMAL(38, 9)), 6) AS uo_tm_24hr FROM uo_tm AS io LEFT JOIN uo_tm AS iosum ON io.stay_id = iosum.stay_id diff --git a/mimic-iv/concepts_duckdb/organfailure/kdigo_uo.sql b/mimic-iv/concepts_duckdb/organfailure/kdigo_uo.sql index 641f536e..f2d15790 100644 --- a/mimic-iv/concepts_duckdb/organfailure/kdigo_uo.sql +++ b/mimic-iv/concepts_duckdb/organfailure/kdigo_uo.sql @@ -38,20 +38,29 @@ WITH uo_stg1 AS ( ORDER BY seconds_since_admit NULLS FIRST RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW ) AS urineoutput_24hr, - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 21600 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 21600 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_6hr, - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 43200 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 43200 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_12hr, - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_24hr FROM uo_stg1 ) diff --git a/mimic-iv/concepts_postgres/demographics/icustay_detail.sql b/mimic-iv/concepts_postgres/demographics/icustay_detail.sql index f20dbd77..1a4c55c7 100644 --- a/mimic-iv/concepts_postgres/demographics/icustay_detail.sql +++ b/mimic-iv/concepts_postgres/demographics/icustay_detail.sql @@ -21,7 +21,7 @@ SELECT ie.intime AS icu_intime, ie.outtime AS icu_outtime, ROUND( - CAST(CAST(CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', ie.outtime) - DATE_TRUNC('hour', ie.intime)) / 3600 AS BIGINT) AS DOUBLE PRECISION) / 24.0 AS DECIMAL), + CAST(CAST(CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', ie.outtime) - DATE_TRUNC('hour', ie.intime)) / 3600 AS BIGINT) AS DOUBLE PRECISION) / 24.0 AS DECIMAL(38, 9)), 2 ) AS los_icu, DENSE_RANK() OVER (PARTITION BY ie.hadm_id ORDER BY ie.intime NULLS FIRST) AS icustay_seq, /* first ICU stay *for the current hospitalization* */ diff --git a/mimic-iv/concepts_postgres/demographics/weight_durations.sql b/mimic-iv/concepts_postgres/demographics/weight_durations.sql index cca044fe..4edf740b 100644 --- a/mimic-iv/concepts_postgres/demographics/weight_durations.sql +++ b/mimic-iv/concepts_postgres/demographics/weight_durations.sql @@ -5,13 +5,14 @@ WITH wt_stg AS ( SELECT c.stay_id, c.charttime, - CASE WHEN c.itemid = 226512 THEN 'admit' ELSE 'daily' END AS weight_type, /* TODO: eliminate obvious outliers if there is a reasonable weight */ - c.valuenum AS weight + CASE WHEN c.itemid = 226512 THEN 'admit' ELSE 'daily' END AS weight_type, + ROUND(CAST(c.valuenum AS DECIMAL(38, 9)), 3) AS weight FROM mimiciv_icu.chartevents AS c WHERE NOT c.valuenum IS NULL AND c.itemid IN (226512, /* Admit Wt */224639 /* Daily Weight */) AND c.valuenum > 0 + AND c.valuenum < 1500 ), wt_stg1 /* assign ascending row number */ AS ( SELECT stay_id, diff --git a/mimic-iv/concepts_postgres/firstday/first_day_height.sql b/mimic-iv/concepts_postgres/firstday/first_day_height.sql index 14f765f9..92da9a24 100644 --- a/mimic-iv/concepts_postgres/firstday/first_day_height.sql +++ b/mimic-iv/concepts_postgres/firstday/first_day_height.sql @@ -4,7 +4,7 @@ DROP TABLE IF EXISTS mimiciv_derived.first_day_height; CREATE TABLE mimiciv_deri SELECT ie.subject_id, ie.stay_id, - ROUND(CAST(AVG(height) AS DECIMAL), 2) AS height + ROUND(CAST(AVG(height) AS DECIMAL(38, 9)), 2) AS height FROM mimiciv_icu.icustays AS ie LEFT JOIN mimiciv_derived.height AS ht ON ie.stay_id = ht.stay_id diff --git a/mimic-iv/concepts_postgres/measurement/bg.sql b/mimic-iv/concepts_postgres/measurement/bg.sql index 994c4bf1..a1d02fd5 100644 --- a/mimic-iv/concepts_postgres/measurement/bg.sql +++ b/mimic-iv/concepts_postgres/measurement/bg.sql @@ -115,7 +115,7 @@ WITH bg AS ( ), stg2 AS ( SELECT bg.*, - ROW_NUMBER() OVER (PARTITION BY bg.subject_id, bg.charttime ORDER BY s1.charttime DESC NULLS LAST) AS lastrowspo2, + ROW_NUMBER() OVER (PARTITION BY bg.specimen_id ORDER BY s1.charttime DESC NULLS LAST) AS lastrowspo2, s1.spo2 FROM bg LEFT JOIN stg_spo2 AS s1 @@ -126,7 +126,7 @@ WITH bg AS ( ), stg3 AS ( SELECT bg.*, - ROW_NUMBER() OVER (PARTITION BY bg.subject_id, bg.charttime ORDER BY s2.charttime DESC NULLS LAST) AS lastrowfio2, + ROW_NUMBER() OVER (PARTITION BY bg.specimen_id ORDER BY s2.charttime DESC NULLS LAST) AS lastrowfio2, s2.fio2_chartevents FROM stg2 AS bg LEFT JOIN stg_fio2 AS s2 diff --git a/mimic-iv/concepts_postgres/measurement/blood_differential.sql b/mimic-iv/concepts_postgres/measurement/blood_differential.sql index 8e339579..73f906e1 100644 --- a/mimic-iv/concepts_postgres/measurement/blood_differential.sql +++ b/mimic-iv/concepts_postgres/measurement/blood_differential.sql @@ -109,7 +109,7 @@ SELECT WHEN basophils_abs IS NULL AND NOT basophils IS NULL AND impute_abs = 1 THEN CAST(basophils * wbc AS DOUBLE PRECISION) / 100 ELSE basophils_abs - END AS DECIMAL), + END AS DECIMAL(38, 9)), 4 ) AS basophils_abs, ROUND( @@ -117,7 +117,7 @@ SELECT WHEN eosinophils_abs IS NULL AND NOT eosinophils IS NULL AND impute_abs = 1 THEN CAST(eosinophils * wbc AS DOUBLE PRECISION) / 100 ELSE eosinophils_abs - END AS DECIMAL), + END AS DECIMAL(38, 9)), 4 ) AS eosinophils_abs, ROUND( @@ -125,7 +125,7 @@ SELECT WHEN lymphocytes_abs IS NULL AND NOT lymphocytes IS NULL AND impute_abs = 1 THEN CAST(lymphocytes * wbc AS DOUBLE PRECISION) / 100 ELSE lymphocytes_abs - END AS DECIMAL), + END AS DECIMAL(38, 9)), 4 ) AS lymphocytes_abs, ROUND( @@ -133,7 +133,7 @@ SELECT WHEN monocytes_abs IS NULL AND NOT monocytes IS NULL AND impute_abs = 1 THEN CAST(monocytes * wbc AS DOUBLE PRECISION) / 100 ELSE monocytes_abs - END AS DECIMAL), + END AS DECIMAL(38, 9)), 4 ) AS monocytes_abs, ROUND( @@ -141,7 +141,7 @@ SELECT WHEN neutrophils_abs IS NULL AND NOT neutrophils IS NULL AND impute_abs = 1 THEN CAST(neutrophils * wbc AS DOUBLE PRECISION) / 100 ELSE neutrophils_abs - END AS DECIMAL), + END AS DECIMAL(38, 9)), 4 ) AS neutrophils_abs, basophils, diff --git a/mimic-iv/concepts_postgres/measurement/height.sql b/mimic-iv/concepts_postgres/measurement/height.sql index e5f33b76..51242417 100644 --- a/mimic-iv/concepts_postgres/measurement/height.sql +++ b/mimic-iv/concepts_postgres/measurement/height.sql @@ -6,7 +6,7 @@ WITH ht_in AS ( c.subject_id, c.stay_id, c.charttime, /* Ensure that all heights are in centimeters */ - ROUND(CAST(c.valuenum * 2.54 AS DECIMAL), 2) AS height, + ROUND(CAST(c.valuenum * 2.54 AS DECIMAL(38, 9)), 2) AS height, c.valuenum AS height_orig FROM mimiciv_icu.chartevents AS c WHERE @@ -16,7 +16,7 @@ WITH ht_in AS ( c.subject_id, c.stay_id, c.charttime, /* Ensure that all heights are in centimeters */ - ROUND(CAST(c.valuenum AS DECIMAL), 2) AS height + ROUND(CAST(c.valuenum AS DECIMAL(38, 9)), 2) AS height FROM mimiciv_icu.chartevents AS c WHERE NOT c.valuenum IS NULL AND /* Height cm */ c.itemid = 226730 diff --git a/mimic-iv/concepts_postgres/measurement/urine_output_rate.sql b/mimic-iv/concepts_postgres/measurement/urine_output_rate.sql index a5856578..4cdc9ef7 100644 --- a/mimic-iv/concepts_postgres/measurement/urine_output_rate.sql +++ b/mimic-iv/concepts_postgres/measurement/urine_output_rate.sql @@ -40,13 +40,16 @@ WITH tm AS ( ELSE NULL END ) AS urineoutput_6hr, - CAST(SUM( - CASE - WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', io.charttime) - DATE_TRUNC('hour', iosum.charttime)) / 3600 AS BIGINT) <= 5 - THEN iosum.tm_since_last_uo - ELSE NULL - END - ) AS DOUBLE PRECISION) / 60.0 AS uo_tm_6hr, + ROUND( + CAST(CAST(SUM( + CASE + WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', io.charttime) - DATE_TRUNC('hour', iosum.charttime)) / 3600 AS BIGINT) <= 5 + THEN iosum.tm_since_last_uo + ELSE NULL + END + ) AS DOUBLE PRECISION) / 60.0 AS DECIMAL(38, 9)), + 6 + ) AS uo_tm_6hr, SUM( CASE WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', io.charttime) - DATE_TRUNC('hour', iosum.charttime)) / 3600 AS BIGINT) <= 11 @@ -54,15 +57,21 @@ WITH tm AS ( ELSE NULL END ) AS urineoutput_12hr, - CAST(SUM( - CASE - WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', io.charttime) - DATE_TRUNC('hour', iosum.charttime)) / 3600 AS BIGINT) <= 11 - THEN iosum.tm_since_last_uo - ELSE NULL - END - ) AS DOUBLE PRECISION) / 60.0 AS uo_tm_12hr, /* 24 hours */ + ROUND( + CAST(CAST(SUM( + CASE + WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', io.charttime) - DATE_TRUNC('hour', iosum.charttime)) / 3600 AS BIGINT) <= 11 + THEN iosum.tm_since_last_uo + ELSE NULL + END + ) AS DOUBLE PRECISION) / 60.0 AS DECIMAL(38, 9)), + 6 + ) AS uo_tm_12hr, /* 24 hours */ SUM(iosum.urineoutput) AS urineoutput_24hr, - CAST(SUM(iosum.tm_since_last_uo) AS DOUBLE PRECISION) / 60.0 AS uo_tm_24hr + ROUND( + CAST(CAST(SUM(iosum.tm_since_last_uo) AS DOUBLE PRECISION) / 60.0 AS DECIMAL(38, 9)), + 6 + ) AS uo_tm_24hr FROM uo_tm AS io /* this join gives you all UO measurements over a 24 hour period */ LEFT JOIN uo_tm AS iosum @@ -88,7 +97,7 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_6hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_6hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) END AS uo_mlkghr_6hr, @@ -97,7 +106,7 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_12hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_12hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) END AS uo_mlkghr_12hr, @@ -106,13 +115,13 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_24hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_24hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) END AS uo_mlkghr_24hr, /* time of earliest UO measurement that was used to calculate the rate */ - ROUND(CAST(uo_tm_6hr AS DECIMAL), 2) AS uo_tm_6hr, - ROUND(CAST(uo_tm_12hr AS DECIMAL), 2) AS uo_tm_12hr, - ROUND(CAST(uo_tm_24hr AS DECIMAL), 2) AS uo_tm_24hr + ROUND(CAST(uo_tm_6hr AS DECIMAL(38, 9)), 2) AS uo_tm_6hr, + ROUND(CAST(uo_tm_12hr AS DECIMAL(38, 9)), 2) AS uo_tm_12hr, + ROUND(CAST(uo_tm_24hr AS DECIMAL(38, 9)), 2) AS uo_tm_24hr FROM ur_stg AS ur LEFT JOIN mimiciv_derived.weight_durations AS wd ON ur.stay_id = wd.stay_id diff --git a/mimic-iv/concepts_postgres/measurement/vitalsign.sql b/mimic-iv/concepts_postgres/measurement/vitalsign.sql index 47c397cd..4646ccb0 100644 --- a/mimic-iv/concepts_postgres/measurement/vitalsign.sql +++ b/mimic-iv/concepts_postgres/measurement/vitalsign.sql @@ -45,7 +45,7 @@ SELECT WHEN itemid IN (223762) AND valuenum > 10 AND valuenum < 50 THEN valuenum END - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 2 ) AS temperature, MAX(CASE WHEN itemid = 224642 THEN value END) AS temperature_site, diff --git a/mimic-iv/concepts_postgres/medication/norepinephrine_equivalent_dose.sql b/mimic-iv/concepts_postgres/medication/norepinephrine_equivalent_dose.sql index b63a0ebe..883e6d88 100644 --- a/mimic-iv/concepts_postgres/medication/norepinephrine_equivalent_dose.sql +++ b/mimic-iv/concepts_postgres/medication/norepinephrine_equivalent_dose.sql @@ -6,7 +6,7 @@ SELECT starttime, endtime, /* calculate the dose */ /* all sources are in mcg/kg/min, */ /* except vasopressin which is in units/hour */ ROUND( - CAST(COALESCE(norepinephrine, 0) + COALESCE(epinephrine, 0) + COALESCE(CAST(phenylephrine AS DOUBLE PRECISION) / 10, 0) + COALESCE(CAST(dopamine AS DOUBLE PRECISION) / 100, 0) + /* + metaraminol/8 -- metaraminol not used in BIDMC */ COALESCE(CAST(vasopressin * 2.5 AS DOUBLE PRECISION) / 60, 0) AS DECIMAL), + CAST(COALESCE(norepinephrine, 0) + COALESCE(epinephrine, 0) + COALESCE(CAST(phenylephrine AS DOUBLE PRECISION) / 10, 0) + COALESCE(CAST(dopamine AS DOUBLE PRECISION) / 100, 0) + /* + metaraminol/8 -- metaraminol not used in BIDMC */ COALESCE(CAST(vasopressin * 2.5 AS DOUBLE PRECISION) / 60, 0) AS DECIMAL(38, 9)), 4 ) AS norepinephrine_equivalent_dose FROM mimiciv_derived.vasoactive_agent diff --git a/mimic-iv/concepts_postgres/organfailure/kdigo_uo.sql b/mimic-iv/concepts_postgres/organfailure/kdigo_uo.sql index 22466fa2..e514dcc0 100644 --- a/mimic-iv/concepts_postgres/organfailure/kdigo_uo.sql +++ b/mimic-iv/concepts_postgres/organfailure/kdigo_uo.sql @@ -34,20 +34,29 @@ WITH uo_stg1 AS ( ORDER BY seconds_since_admit NULLS FIRST RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW ) AS urineoutput_24hr, /* repeat the summations using the hours_since_previous_row column */ /* this gives us the amount of time the UO was calculated over */ - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 21600 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 21600 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_6hr, - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 43200 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 43200 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_12hr, - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_24hr FROM uo_stg1 ) @@ -63,7 +72,7 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_6hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_6hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) ELSE NULL @@ -73,7 +82,7 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_12hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_12hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) ELSE NULL @@ -83,7 +92,7 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_24hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_24hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) ELSE NULL diff --git a/mimic-iv/concepts_postgres/organfailure/meld.sql b/mimic-iv/concepts_postgres/organfailure/meld.sql index 9d2f5130..f6c28ebb 100644 --- a/mimic-iv/concepts_postgres/organfailure/meld.sql +++ b/mimic-iv/concepts_postgres/organfailure/meld.sql @@ -83,7 +83,7 @@ WITH cohort AS ( creatinine_score + bilirubin_score + inr_score ) > 4 THEN 40.0 - ELSE ROUND(CAST(creatinine_score + bilirubin_score + inr_score AS DECIMAL), 1) * 10 + ELSE ROUND(CAST(creatinine_score + bilirubin_score + inr_score AS DECIMAL(38, 9)), 1) * 10 END AS meld_initial FROM score ) From 0981068379007d102a519c39d6745c3bec79b77a Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 6 Jul 2026 08:00:42 -0400 Subject: [PATCH 09/26] fix mismatch in sorts due to differing collation --- .github/workflows/build-psql.yml | 3 +++ .github/workflows/concepts-psql.yml | 3 +++ .github/workflows/equivalence.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.github/workflows/build-psql.yml b/.github/workflows/build-psql.yml index cebe1d8c..2e03054c 100644 --- a/.github/workflows/build-psql.yml +++ b/.github/workflows/build-psql.yml @@ -14,6 +14,9 @@ jobs: image: postgres:17 env: POSTGRES_PASSWORD: postgres + # Force byte-order (C) collation so string ordering matches DuckDB's + # binary sort. + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" options: >- --health-cmd pg_isready --health-interval 10s diff --git a/.github/workflows/concepts-psql.yml b/.github/workflows/concepts-psql.yml index 320342eb..5a5c4ec5 100644 --- a/.github/workflows/concepts-psql.yml +++ b/.github/workflows/concepts-psql.yml @@ -15,6 +15,9 @@ jobs: image: postgres:17 env: POSTGRES_PASSWORD: postgres + # Force byte-order (C) collation so string ordering matches DuckDB's + # binary sort. + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" options: >- --health-cmd pg_isready --health-interval 10s diff --git a/.github/workflows/equivalence.yml b/.github/workflows/equivalence.yml index deee28df..da16f718 100644 --- a/.github/workflows/equivalence.yml +++ b/.github/workflows/equivalence.yml @@ -18,6 +18,9 @@ jobs: image: postgres:17 env: POSTGRES_PASSWORD: postgres + # Force byte-order (C) collation so string ordering matches DuckDB's + # binary sort. + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" options: >- --health-cmd pg_isready --health-interval 10s From 3c33eb250dda67a0e3b2ed2af2c755ba02212132 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 6 Jul 2026 08:08:04 -0400 Subject: [PATCH 10/26] remove tables that should now match --- .github/workflows/equivalence.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/equivalence.yml b/.github/workflows/equivalence.yml index da16f718..c1750736 100644 --- a/.github/workflows/equivalence.yml +++ b/.github/workflows/equivalence.yml @@ -6,7 +6,7 @@ on: env: # Tables with known cross-engine differences - IGNORE_TABLES: "first_day_gcs,suspicion_of_infection,oxygen_delivery,bg,first_day_bg,first_day_bg_art,kdigo_uo,kdigo_stages,urine_output_rate" + IGNORE_TABLES: "first_day_gcs,suspicion_of_infection,oxygen_delivery,bg,first_day_bg,first_day_bg_art" jobs: diff: From cea682e4896ea3033140dc5855e159233e121ce3 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 6 Jul 2026 16:47:21 -0400 Subject: [PATCH 11/26] update cache key and stop mysql job from modifying cache folder --- .github/actions/download-demo/action.yml | 7 ++++--- .github/workflows/mysql.yml | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/actions/download-demo/action.yml b/.github/actions/download-demo/action.yml index 698df4ce..ff5407d3 100644 --- a/.github/actions/download-demo/action.yml +++ b/.github/actions/download-demo/action.yml @@ -10,8 +10,6 @@ inputs: runs: using: "composite" steps: - # The demo datasets are version-pinned and immutable, so a static key never - # goes stale - name: Restore demo data from cache id: cache uses: actions/cache@v4 @@ -20,7 +18,10 @@ runs: ${{ inputs.dest }}/hosp ${{ inputs.dest }}/icu ${{ inputs.dest }}/ed - key: mimic-iv-demo-2.2 + # -v2: the original key was poisoned with empty dirs by a workflow that + # moved CSVs out of the cached hosp/, icu/, ed/ folders before the + # post-job cache save. + key: mimic-iv-demo-2.2-v2 - name: Download demo data from PhysioNet if: steps.cache.outputs.cache-hit != 'true' diff --git a/.github/workflows/mysql.yml b/.github/workflows/mysql.yml index 73b56cc8..8a6e8b69 100644 --- a/.github/workflows/mysql.yml +++ b/.github/workflows/mysql.yml @@ -27,7 +27,9 @@ jobs: # Flatten every nested CSV into the working directory, where the mysql # load.sql scripts expect them (e.g. LOAD DATA INFILE 'admissions.csv'). # Using find keeps this robust to how deeply the files are nested. - find . -mindepth 2 -name '*.csv.gz' -exec mv -t ./ {} + + # NB: copy (not move) so hosp/, icu/ and ed/ retain their contents + # and the cache is preserved. + find . -mindepth 2 -name '*.csv.gz' -exec cp -t ./ {} + gzip -d *.csv.gz - name: Start MySQL service From 15ae2cfd16964274c8f8f1860ab1c6e77825afa4 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 6 Jul 2026 16:55:11 -0400 Subject: [PATCH 12/26] fix non-determinism in o2 flow by taking highest value --- mimic-iv/concepts/measurement/oxygen_delivery.sql | 13 ++++++++++--- .../concepts_duckdb/measurement/oxygen_delivery.sql | 4 ++-- .../measurement/oxygen_delivery.sql | 12 +++++++++--- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/mimic-iv/concepts/measurement/oxygen_delivery.sql b/mimic-iv/concepts/measurement/oxygen_delivery.sql index fc2bbf7f..e76f6303 100644 --- a/mimic-iv/concepts/measurement/oxygen_delivery.sql +++ b/mimic-iv/concepts/measurement/oxygen_delivery.sql @@ -34,10 +34,15 @@ WITH ce_stg1 AS ( , valuenum , valueuom -- retain only 1 row per charttime - -- prioritizing the last documented value + -- prioritizing (1) the last documented value + -- and (2) the highest value -- primarily used to subselect o2 flows + -- when storetime ties (e.g. an o2 flow 223834 and a bipap o2 flow + -- 227582 documented together), prefer the highest value so the result + -- is deterministic across SQL engines , ROW_NUMBER() OVER ( - PARTITION BY subject_id, charttime, itemid ORDER BY storetime DESC + PARTITION BY subject_id, charttime, itemid + ORDER BY storetime DESC, valuenum DESC ) AS rn FROM ce_stg1 ce ) @@ -62,7 +67,9 @@ WITH ce_stg1 AS ( , itemid , value AS o2_device , ROW_NUMBER() OVER ( - PARTITION BY subject_id, charttime, itemid ORDER BY value + PARTITION BY subject_id, charttime, itemid + -- sort so the result is deterministic across SQL engines + ORDER BY storetime DESC NULLS LAST, value DESC NULLS LAST ) AS rn FROM `physionet-data.mimiciv_icu.chartevents` WHERE itemid = 226732 -- oxygen delivery device(s) diff --git a/mimic-iv/concepts_duckdb/measurement/oxygen_delivery.sql b/mimic-iv/concepts_duckdb/measurement/oxygen_delivery.sql index e3a5b955..0bfef488 100644 --- a/mimic-iv/concepts_duckdb/measurement/oxygen_delivery.sql +++ b/mimic-iv/concepts_duckdb/measurement/oxygen_delivery.sql @@ -22,7 +22,7 @@ WITH ce_stg1 AS ( value, valuenum, valueuom, - ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY storetime DESC) AS rn + ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY storetime DESC, valuenum DESC) AS rn FROM ce_stg1 AS ce ), o2 AS ( SELECT @@ -31,7 +31,7 @@ WITH ce_stg1 AS ( charttime, itemid, value AS o2_device, - ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY value NULLS FIRST) AS rn + ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY storetime DESC, value DESC) AS rn FROM mimiciv_icu.chartevents WHERE itemid = 226732 diff --git a/mimic-iv/concepts_postgres/measurement/oxygen_delivery.sql b/mimic-iv/concepts_postgres/measurement/oxygen_delivery.sql index b23f7c53..d3594237 100644 --- a/mimic-iv/concepts_postgres/measurement/oxygen_delivery.sql +++ b/mimic-iv/concepts_postgres/measurement/oxygen_delivery.sql @@ -26,8 +26,11 @@ WITH ce_stg1 AS ( itemid, value, valuenum, - valueuom, /* retain only 1 row per charttime */ /* prioritizing the last documented value */ /* primarily used to subselect o2 flows */ - ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY storetime DESC NULLS LAST) AS rn + valueuom, /* retain only 1 row per charttime */ /* prioritizing (1) the last documented value */ /* and (2) the highest value */ /* primarily used to subselect o2 flows */ /* when storetime ties (e.g. an o2 flow 223834 and a bipap o2 flow */ /* 227582 documented together), prefer the highest value so the result */ /* is deterministic across SQL engines */ + ROW_NUMBER() OVER ( + PARTITION BY subject_id, charttime, itemid + ORDER BY storetime DESC NULLS LAST, valuenum DESC NULLS LAST + ) AS rn FROM ce_stg1 AS ce ), o2 AS ( /* The below ITEMID can have multiple entries for charttime/storetime */ /* These are valid entries, and should be retained in derived tables. */ /* 224181 -- Small Volume Neb Drug #1 | Respiratory | Text */ /* , 227570 -- Small Volume Neb Drug/Dose #1 | Respiratory | Text */ /* , 224833 -- SBT Deferred | Respiratory | Text */ /* , 224716 -- SBT Stopped | Respiratory | Text */ /* , 224740 -- RSBI Deferred | Respiratory | Text */ /* , 224829 -- Trach Tube Type | Respiratory | Text */ /* , 226732 -- O2 Delivery Device(s) | Respiratory | Text */ /* , 226873 -- Inspiratory Ratio | Respiratory | Numeric */ /* , 226871 -- Expiratory Ratio | Respiratory | Numeric */ /* maximum of 4 o2 devices on at once */ @@ -37,7 +40,10 @@ WITH ce_stg1 AS ( charttime, itemid, value AS o2_device, - ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY value NULLS FIRST) AS rn + ROW_NUMBER() OVER ( + PARTITION BY subject_id, charttime, itemid + ORDER BY storetime DESC NULLS LAST, value DESC NULLS LAST + ) AS rn FROM mimiciv_icu.chartevents WHERE itemid = 226732 /* oxygen delivery device(s) */ From a399a08a2454636dc9930d6834356c37e9b7943d Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 6 Jul 2026 17:01:15 -0400 Subject: [PATCH 13/26] index concepts so sofa.sql builds in a reasonable amount of time --- .../postgres-concept-index.sql | 74 +++++++++++++++++++ .../postgres-make-concepts.sql | 10 ++- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 mimic-iv/concepts_postgres/postgres-concept-index.sql diff --git a/mimic-iv/concepts_postgres/postgres-concept-index.sql b/mimic-iv/concepts_postgres/postgres-concept-index.sql new file mode 100644 index 00000000..8decebc7 --- /dev/null +++ b/mimic-iv/concepts_postgres/postgres-concept-index.sql @@ -0,0 +1,74 @@ +---------------------------------------------------- +---------------------------------------------------- +-- Indexes for the mimiciv_derived concept tables -- +---------------------------------------------------- +---------------------------------------------------- + +-- The score/firstday concepts join hourly windows against the measurement +-- and medication concepts using (stay_id/hadm_id/subject_id, charttime) +-- range predicates. Without these indexes PostgreSQL falls back to +-- merge/hash joins that materialise hundreds of millions of intermediate +-- rows. With them the joins become per-hour index lookups and the same +-- scripts run in minutes. + +SET search_path TO mimiciv_derived; + +DROP INDEX IF EXISTS icustay_hourly_idx01; +CREATE INDEX icustay_hourly_idx01 + ON icustay_hourly (stay_id); + +-- measurement concepts joined on stay_id + charttime + +DROP INDEX IF EXISTS vitalsign_idx01; +CREATE INDEX vitalsign_idx01 + ON vitalsign (stay_id, charttime); + +DROP INDEX IF EXISTS gcs_idx01; +CREATE INDEX gcs_idx01 + ON gcs (stay_id, charttime); + +DROP INDEX IF EXISTS urine_output_rate_idx01; +CREATE INDEX urine_output_rate_idx01 + ON urine_output_rate (stay_id, charttime); + +-- measurement concepts joined on hadm_id + charttime + +DROP INDEX IF EXISTS chemistry_idx01; +CREATE INDEX chemistry_idx01 + ON chemistry (hadm_id, charttime); + +DROP INDEX IF EXISTS complete_blood_count_idx01; +CREATE INDEX complete_blood_count_idx01 + ON complete_blood_count (hadm_id, charttime); + +DROP INDEX IF EXISTS enzyme_idx01; +CREATE INDEX enzyme_idx01 + ON enzyme (hadm_id, charttime); + +-- blood gas joined on subject_id + charttime + +DROP INDEX IF EXISTS bg_idx01; +CREATE INDEX bg_idx01 + ON bg (subject_id, charttime); + +-- ventilation and vasopressor concepts joined on stay_id + starttime/endtime + +DROP INDEX IF EXISTS ventilation_idx01; +CREATE INDEX ventilation_idx01 + ON ventilation (stay_id, starttime, endtime); + +DROP INDEX IF EXISTS epinephrine_idx01; +CREATE INDEX epinephrine_idx01 + ON epinephrine (stay_id, starttime, endtime); + +DROP INDEX IF EXISTS norepinephrine_idx01; +CREATE INDEX norepinephrine_idx01 + ON norepinephrine (stay_id, starttime, endtime); + +DROP INDEX IF EXISTS dopamine_idx01; +CREATE INDEX dopamine_idx01 + ON dopamine (stay_id, starttime, endtime); + +DROP INDEX IF EXISTS dobutamine_idx01; +CREATE INDEX dobutamine_idx01 + ON dobutamine (stay_id, starttime, endtime); diff --git a/mimic-iv/concepts_postgres/postgres-make-concepts.sql b/mimic-iv/concepts_postgres/postgres-make-concepts.sql index 3bbed1e3..97e3ea8b 100644 --- a/mimic-iv/concepts_postgres/postgres-make-concepts.sql +++ b/mimic-iv/concepts_postgres/postgres-make-concepts.sql @@ -1,8 +1,8 @@ \echo '' \echo '===' -\echo 'Beginning to create materialized views for MIMIC database.' -\echo 'Any notices of the form "NOTICE: materialized view "XXXXXX" does not exist" can be ignored.' -\echo 'The scripts drop views before creating them, and these notices indicate nothing existed prior to creating the view.' +\echo 'Beginning to create derived concepts for MIMIC-IV.' +\echo 'Any notices of the form "NOTICE: table "XXXXXX" does not exist" can be ignored.' +\echo 'The scripts drop tables before creating them, and these notices indicate nothing existed.' \echo '===' \echo '' @@ -80,6 +80,10 @@ SET search_path TO mimiciv_derived, mimiciv_hosp, mimiciv_icu, mimiciv_ed; \i organfailure/kdigo_creatinine.sql \i organfailure/meld.sql +-- indexes on the concept tables above, required for the hourly score +-- queries (sofa, apsiii, ...) to run in minutes rather than hours +\i postgres-concept-index.sql + -- score \i score/apsiii.sql \i score/lods.sql From 39ae9c2d2101b4f7be29a18f4960a7f62f4329f0 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 6 Jul 2026 17:05:18 -0400 Subject: [PATCH 14/26] revert key change --- .github/actions/download-demo/action.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/actions/download-demo/action.yml b/.github/actions/download-demo/action.yml index ff5407d3..34525d69 100644 --- a/.github/actions/download-demo/action.yml +++ b/.github/actions/download-demo/action.yml @@ -18,10 +18,7 @@ runs: ${{ inputs.dest }}/hosp ${{ inputs.dest }}/icu ${{ inputs.dest }}/ed - # -v2: the original key was poisoned with empty dirs by a workflow that - # moved CSVs out of the cached hosp/, icu/, ed/ folders before the - # post-job cache save. - key: mimic-iv-demo-2.2-v2 + key: mimic-iv-demo-2.2 - name: Download demo data from PhysioNet if: steps.cache.outputs.cache-hit != 'true' From fb98aa7ee9dc3a460497d854b8a0ea1890b9a5f1 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 6 Jul 2026 18:10:31 -0400 Subject: [PATCH 15/26] make first day GCS deterministic by sorting by charttime/storetime --- mimic-iv/concepts/firstday/first_day_gcs.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mimic-iv/concepts/firstday/first_day_gcs.sql b/mimic-iv/concepts/firstday/first_day_gcs.sql index b7469852..bef61c1f 100644 --- a/mimic-iv/concepts/firstday/first_day_gcs.sql +++ b/mimic-iv/concepts/firstday/first_day_gcs.sql @@ -21,10 +21,13 @@ WITH gcs_final AS ( , g.gcs_unable -- This sorts the data by GCS -- rn = 1 is the the lowest total GCS value + -- tie-break on charttime/storetime (unique per stay) so the component columns + -- are deterministic across SQL engines when multiple measurements + -- share the same minimum GCS , ROW_NUMBER() OVER ( PARTITION BY g.stay_id - ORDER BY g.gcs + ORDER BY g.gcs ASC NULLS LAST, g.charttime DESC NULLS LAST, g.storetime DESC NULLS LAST ) AS gcs_seq FROM `physionet-data.mimiciv_icu.icustays` ie -- Only get data for the first 24 hours From 88c9104979e8429dac5ac38eaa908d0596c4a593 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 6 Jul 2026 18:10:47 -0400 Subject: [PATCH 16/26] transpile first_day_gcs --- mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql | 2 +- mimic-iv/concepts_postgres/firstday/first_day_gcs.sql | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql b/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql index d11e2f9a..13f29b90 100644 --- a/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql +++ b/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql @@ -9,7 +9,7 @@ WITH gcs_final AS ( g.gcs_verbal, g.gcs_eyes, g.gcs_unable, - ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs NULLS FIRST) AS gcs_seq + ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs ASC, g.charttime DESC, g.storetime DESC) AS gcs_seq FROM mimiciv_icu.icustays AS ie LEFT JOIN mimiciv_derived.gcs AS g ON ie.stay_id = g.stay_id diff --git a/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql b/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql index 67155d82..2d7424f3 100644 --- a/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql +++ b/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql @@ -9,8 +9,11 @@ WITH gcs_final AS ( g.gcs_motor, g.gcs_verbal, g.gcs_eyes, - g.gcs_unable, /* This sorts the data by GCS */ /* rn = 1 is the the lowest total GCS value */ - ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs NULLS FIRST) AS gcs_seq + g.gcs_unable, /* This sorts the data by GCS */ /* rn = 1 is the the lowest total GCS value */ /* tie-break on charttime/storetime (unique per stay) so the component columns */ /* are deterministic across SQL engines when multiple measurements */ /* share the same minimum GCS */ + ROW_NUMBER() OVER ( + PARTITION BY g.stay_id + ORDER BY g.gcs ASC, g.charttime DESC NULLS LAST, g.storetime DESC NULLS LAST + ) AS gcs_seq FROM mimiciv_icu.icustays AS ie /* Only get data for the first 24 hours */ LEFT JOIN mimiciv_derived.gcs AS g From 5a45342850301651fdb2e23596a093875276d73f Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Mon, 6 Jul 2026 18:56:53 -0400 Subject: [PATCH 17/26] ensure bg is deterministic --- mimic-iv/concepts/measurement/bg.sql | 25 +++++----- mimic-iv/concepts_duckdb/measurement/bg.sql | 45 +++++++++--------- mimic-iv/concepts_postgres/measurement/bg.sql | 47 ++++++++++--------- 3 files changed, 62 insertions(+), 55 deletions(-) diff --git a/mimic-iv/concepts/measurement/bg.sql b/mimic-iv/concepts/measurement/bg.sql index a5b393ef..41ffbbab 100644 --- a/mimic-iv/concepts/measurement/bg.sql +++ b/mimic-iv/concepts/measurement/bg.sql @@ -193,18 +193,19 @@ SELECT , pco2 , fio2_chartevents, fio2 , aado2 - -- also calculate AADO2 - , CASE - WHEN po2 IS NULL - OR pco2 IS NULL - THEN NULL - WHEN fio2 IS NOT NULL - -- multiple by 100 because fio2 is in a % but should be a fraction - THEN (fio2 / 100) * (760 - 47) - (pco2 / 0.8) - po2 - WHEN fio2_chartevents IS NOT NULL - THEN (fio2_chartevents / 100) * (760 - 47) - (pco2 / 0.8) - po2 - ELSE NULL - END AS aado2_calc + -- also calculate AADO2, rounded to 4 decimal places + , ROUND(CAST( + CASE + WHEN po2 IS NULL + OR pco2 IS NULL + THEN NULL + WHEN fio2 IS NOT NULL + -- multiple by 100 because fio2 is in a % but should be a fraction + THEN (fio2 / 100) * (760 - 47) - (pco2 / 0.8) - po2 + WHEN fio2_chartevents IS NOT NULL + THEN (fio2_chartevents / 100) * (760 - 47) - (pco2 / 0.8) - po2 + ELSE NULL + END AS NUMERIC), 4) AS aado2_calc , CASE WHEN po2 IS NULL THEN NULL diff --git a/mimic-iv/concepts_duckdb/measurement/bg.sql b/mimic-iv/concepts_duckdb/measurement/bg.sql index 9fc631da..79286421 100644 --- a/mimic-iv/concepts_duckdb/measurement/bg.sql +++ b/mimic-iv/concepts_duckdb/measurement/bg.sql @@ -143,27 +143,30 @@ SELECT fio2_chartevents, fio2, aado2, - CASE - WHEN po2 IS NULL OR pco2 IS NULL - THEN NULL - WHEN NOT fio2 IS NULL - THEN ( - fio2 / 100 - ) * ( - 760 - 47 - ) - ( - pco2 / 0.8 - ) - po2 - WHEN NOT fio2_chartevents IS NULL - THEN ( - fio2_chartevents / 100 - ) * ( - 760 - 47 - ) - ( - pco2 / 0.8 - ) - po2 - ELSE NULL - END AS aado2_calc, + ROUND( + CAST(CASE + WHEN po2 IS NULL OR pco2 IS NULL + THEN NULL + WHEN NOT fio2 IS NULL + THEN ( + fio2 / 100 + ) * ( + 760 - 47 + ) - ( + pco2 / 0.8 + ) - po2 + WHEN NOT fio2_chartevents IS NULL + THEN ( + fio2_chartevents / 100 + ) * ( + 760 - 47 + ) - ( + pco2 / 0.8 + ) - po2 + ELSE NULL + END AS DECIMAL(38, 9)), + 4 + ) AS aado2_calc, CASE WHEN po2 IS NULL THEN NULL diff --git a/mimic-iv/concepts_postgres/measurement/bg.sql b/mimic-iv/concepts_postgres/measurement/bg.sql index a1d02fd5..dca59ee3 100644 --- a/mimic-iv/concepts_postgres/measurement/bg.sql +++ b/mimic-iv/concepts_postgres/measurement/bg.sql @@ -148,28 +148,31 @@ SELECT pco2, fio2_chartevents, fio2, - aado2, /* also calculate AADO2 */ - CASE - WHEN po2 IS NULL OR pco2 IS NULL - THEN NULL - WHEN NOT fio2 IS NULL - THEN ( - CAST(fio2 AS DOUBLE PRECISION) / 100 - ) * ( - 760 - 47 - ) - ( - CAST(pco2 AS DOUBLE PRECISION) / 0.8 - ) - po2 - WHEN NOT fio2_chartevents IS NULL - THEN ( - CAST(fio2_chartevents AS DOUBLE PRECISION) / 100 - ) * ( - 760 - 47 - ) - ( - CAST(pco2 AS DOUBLE PRECISION) / 0.8 - ) - po2 - ELSE NULL - END AS aado2_calc, + aado2, /* also calculate AADO2, rounded to 4 decimal places */ + ROUND( + CAST(CASE + WHEN po2 IS NULL OR pco2 IS NULL + THEN NULL + WHEN NOT fio2 IS NULL + THEN ( + CAST(fio2 AS DOUBLE PRECISION) / 100 + ) * ( + 760 - 47 + ) - ( + CAST(pco2 AS DOUBLE PRECISION) / 0.8 + ) - po2 + WHEN NOT fio2_chartevents IS NULL + THEN ( + CAST(fio2_chartevents AS DOUBLE PRECISION) / 100 + ) * ( + 760 - 47 + ) - ( + CAST(pco2 AS DOUBLE PRECISION) / 0.8 + ) - po2 + ELSE NULL + END AS DECIMAL(38, 9)), + 4 + ) AS aado2_calc, CASE WHEN po2 IS NULL THEN NULL From c07b7a91568177a8b540943237a01548954bad48 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Tue, 7 Jul 2026 00:47:02 -0400 Subject: [PATCH 18/26] fix suspicion of infection to use positive culture, if it exists, indep of ties --- .../sepsis/suspicion_of_infection.sql | 24 ++++++++++++++++--- .../sepsis/suspicion_of_infection.sql | 6 ++--- .../sepsis/suspicion_of_infection.sql | 12 +++++----- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/mimic-iv/concepts/sepsis/suspicion_of_infection.sql b/mimic-iv/concepts/sepsis/suspicion_of_infection.sql index 5b7db295..2fb271b4 100644 --- a/mimic-iv/concepts/sepsis/suspicion_of_infection.sql +++ b/mimic-iv/concepts/sepsis/suspicion_of_infection.sql @@ -10,10 +10,12 @@ WITH ab_tbl AS ( , DATETIME_TRUNC(abx.starttime, DAY) AS antibiotic_date , abx.stoptime AS antibiotic_stoptime -- create a unique identifier for each patient antibiotic + -- hadm_id/stay_id tiebreak keeps the numbering deterministic when + -- multiple antibiotics share the same start/stop time and name , ROW_NUMBER() OVER ( PARTITION BY subject_id - ORDER BY starttime, stoptime, antibiotic + ORDER BY starttime, stoptime, antibiotic, hadm_id, stay_id ) AS ab_id FROM `physionet-data.mimiciv_derived.antibiotic` abx ) @@ -57,10 +59,18 @@ WITH ab_tbl AS ( -- before this abx -- this ensures each antibiotic is only matched to a single culture -- and consequently we have 1 row per antibiotic + -- among cultures at the same time point, prefer a positive culture + -- (positiveculture DESC) so that any positive culture yields + -- positive_culture = 1, per the suspicion of infection criteria; the + -- micro_specimen_id tiebreak makes the selection deterministic , ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me72.chartdate, me72.charttime NULLS LAST + ORDER BY + me72.chartdate + , me72.charttime NULLS LAST + , me72.positiveculture DESC + , me72.micro_specimen_id ) AS micro_seq FROM ab_tbl -- abx taken after culture, but no more than 72 hours after @@ -104,10 +114,18 @@ WITH ab_tbl AS ( -- before this abx -- this ensures each antibiotic is only matched to a single culture -- and consequently we have 1 row per antibiotic + -- among cultures at the same time point, prefer a positive culture + -- (positiveculture DESC) so that any positive culture yields + -- positive_culture = 1, per the suspicion of infection criteria; the + -- micro_specimen_id tiebreak makes the selection deterministic , ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me24.chartdate, me24.charttime NULLS LAST + ORDER BY + me24.chartdate + , me24.charttime NULLS LAST + , me24.positiveculture DESC + , me24.micro_specimen_id ) AS micro_seq FROM ab_tbl -- culture in subsequent 24 hours diff --git a/mimic-iv/concepts_duckdb/sepsis/suspicion_of_infection.sql b/mimic-iv/concepts_duckdb/sepsis/suspicion_of_infection.sql index e2bce86e..f18ccd8f 100644 --- a/mimic-iv/concepts_duckdb/sepsis/suspicion_of_infection.sql +++ b/mimic-iv/concepts_duckdb/sepsis/suspicion_of_infection.sql @@ -11,7 +11,7 @@ WITH ab_tbl AS ( abx.stoptime AS antibiotic_stoptime, ROW_NUMBER() OVER ( PARTITION BY subject_id - ORDER BY starttime NULLS FIRST, stoptime NULLS FIRST, antibiotic NULLS FIRST + ORDER BY starttime NULLS FIRST, stoptime NULLS FIRST, antibiotic NULLS FIRST, hadm_id NULLS FIRST, stay_id NULLS FIRST ) AS ab_id FROM mimiciv_derived.antibiotic AS abx ), me AS ( @@ -44,7 +44,7 @@ WITH ab_tbl AS ( me72.spec_type_desc AS last72_specimen, ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me72.chartdate NULLS FIRST, me72.charttime + ORDER BY me72.chartdate NULLS FIRST, me72.charttime, me72.positiveculture DESC, me72.micro_specimen_id NULLS FIRST ) AS micro_seq FROM ab_tbl LEFT JOIN me AS me72 @@ -73,7 +73,7 @@ WITH ab_tbl AS ( me24.spec_type_desc AS next24_specimen, ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me24.chartdate NULLS FIRST, me24.charttime + ORDER BY me24.chartdate NULLS FIRST, me24.charttime, me24.positiveculture DESC, me24.micro_specimen_id NULLS FIRST ) AS micro_seq FROM ab_tbl LEFT JOIN me AS me24 diff --git a/mimic-iv/concepts_postgres/sepsis/suspicion_of_infection.sql b/mimic-iv/concepts_postgres/sepsis/suspicion_of_infection.sql index 40c548fd..67ff3c07 100644 --- a/mimic-iv/concepts_postgres/sepsis/suspicion_of_infection.sql +++ b/mimic-iv/concepts_postgres/sepsis/suspicion_of_infection.sql @@ -9,10 +9,10 @@ WITH ab_tbl AS ( abx.antibiotic, abx.starttime AS antibiotic_time, /* date is used to match microbiology cultures with only date available */ DATE_TRUNC('day', abx.starttime) AS antibiotic_date, - abx.stoptime AS antibiotic_stoptime, /* create a unique identifier for each patient antibiotic */ + abx.stoptime AS antibiotic_stoptime, /* create a unique identifier for each patient antibiotic */ /* hadm_id/stay_id tiebreak keeps the numbering deterministic when */ /* multiple antibiotics share the same start/stop time and name */ ROW_NUMBER() OVER ( PARTITION BY subject_id - ORDER BY starttime NULLS FIRST, stoptime NULLS FIRST, antibiotic NULLS FIRST + ORDER BY starttime NULLS FIRST, stoptime NULLS FIRST, antibiotic NULLS FIRST, hadm_id NULLS FIRST, stay_id NULLS FIRST ) AS ab_id FROM mimiciv_derived.antibiotic AS abx ), me AS ( @@ -42,10 +42,10 @@ WITH ab_tbl AS ( me72.micro_specimen_id, COALESCE(me72.charttime, CAST(me72.chartdate AS TIMESTAMP)) AS last72_charttime, me72.positiveculture AS last72_positiveculture, - me72.spec_type_desc AS last72_specimen, /* we will use this partition to select the earliest culture */ /* before this abx */ /* this ensures each antibiotic is only matched to a single culture */ /* and consequently we have 1 row per antibiotic */ + me72.spec_type_desc AS last72_specimen, /* we will use this partition to select the earliest culture */ /* before this abx */ /* this ensures each antibiotic is only matched to a single culture */ /* and consequently we have 1 row per antibiotic */ /* among cultures at the same time point, prefer a positive culture */ /* (positiveculture DESC) so that any positive culture yields */ /* positive_culture = 1, per the suspicion of infection criteria; the */ /* micro_specimen_id tiebreak makes the selection deterministic */ ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me72.chartdate NULLS FIRST, me72.charttime + ORDER BY me72.chartdate NULLS FIRST, me72.charttime, me72.positiveculture DESC NULLS LAST, me72.micro_specimen_id NULLS FIRST ) AS micro_seq FROM ab_tbl /* abx taken after culture, but no more than 72 hours after */ @@ -72,10 +72,10 @@ WITH ab_tbl AS ( me24.micro_specimen_id, COALESCE(me24.charttime, CAST(me24.chartdate AS TIMESTAMP)) AS next24_charttime, me24.positiveculture AS next24_positiveculture, - me24.spec_type_desc AS next24_specimen, /* we will use this partition to select the earliest culture */ /* before this abx */ /* this ensures each antibiotic is only matched to a single culture */ /* and consequently we have 1 row per antibiotic */ + me24.spec_type_desc AS next24_specimen, /* we will use this partition to select the earliest culture */ /* before this abx */ /* this ensures each antibiotic is only matched to a single culture */ /* and consequently we have 1 row per antibiotic */ /* among cultures at the same time point, prefer a positive culture */ /* (positiveculture DESC) so that any positive culture yields */ /* positive_culture = 1, per the suspicion of infection criteria; the */ /* micro_specimen_id tiebreak makes the selection deterministic */ ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me24.chartdate NULLS FIRST, me24.charttime + ORDER BY me24.chartdate NULLS FIRST, me24.charttime, me24.positiveculture DESC NULLS LAST, me24.micro_specimen_id NULLS FIRST ) AS micro_seq FROM ab_tbl /* culture in subsequent 24 hours */ From d50c21b995d861251428b3ab52c6bf5dd9a7940b Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Tue, 7 Jul 2026 00:47:17 -0400 Subject: [PATCH 19/26] remove ignore tables from equivalence; all tables should pass --- .github/workflows/equivalence.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/equivalence.yml b/.github/workflows/equivalence.yml index c1750736..5197aaae 100644 --- a/.github/workflows/equivalence.yml +++ b/.github/workflows/equivalence.yml @@ -5,8 +5,8 @@ on: workflow_call: env: - # Tables with known cross-engine differences - IGNORE_TABLES: "first_day_gcs,suspicion_of_infection,oxygen_delivery,bg,first_day_bg,first_day_bg_art" + # If tables are known to be non-equivalent, list them here (comma delimited) to ignore them in the diff. + IGNORE_TABLES: "" jobs: diff: From b2cfb6e049c7acfb419dae53af09a60a185c2765 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Tue, 7 Jul 2026 00:52:10 -0400 Subject: [PATCH 20/26] remove push from ci/mysql/sqlite, add it to pre-release bq --- .github/workflows/ci.yml | 10 ++-------- .github/workflows/mysql.yml | 7 +------ .github/workflows/prerelease_bigquery.yml | 4 ++++ .github/workflows/sqlite.yml | 7 +------ 4 files changed, 8 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70eee0f8..31b34c31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,20 +13,14 @@ on: pull_request_review: types: [submitted] workflow_dispatch: - # TEMP: test on push to branch - push: - branches: - - alistair/more_gh_action_shenanigans jobs: psql: - # TEMP: 'push' added to run on branch push - if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || github.event.review.state == 'approved' + if: github.event_name == 'workflow_dispatch' || github.event.review.state == 'approved' uses: ./.github/workflows/build-psql.yml duckdb: - # TEMP: 'push' added to run on branch push - if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || github.event.review.state == 'approved' + if: github.event_name == 'workflow_dispatch' || github.event.review.state == 'approved' uses: ./.github/workflows/build-duckdb.yml psql-concepts: diff --git a/.github/workflows/mysql.yml b/.github/workflows/mysql.yml index 8a6e8b69..76c465ae 100644 --- a/.github/workflows/mysql.yml +++ b/.github/workflows/mysql.yml @@ -2,16 +2,11 @@ name: mysql demo db build on: pull_request_review: types: [submitted] - # TEMP: test on push to branch - push: - branches: - - alistair/more_gh_action_shenanigans jobs: mimic-iv-mysql: # only run if PR is approved - # TEMP: also run on push - if: github.event_name == 'push' || github.event.review.state == 'approved' + if: github.event.review.state == 'approved' runs-on: ubuntu-22.04 steps: diff --git a/.github/workflows/prerelease_bigquery.yml b/.github/workflows/prerelease_bigquery.yml index 01da3995..afabe84a 100644 --- a/.github/workflows/prerelease_bigquery.yml +++ b/.github/workflows/prerelease_bigquery.yml @@ -5,6 +5,10 @@ name: Pre-release concept build on mimiciv_derived on: release: types: [prereleased] + # TEMP: test on push to branch + push: + branches: + - alistair/more_gh_action_shenanigans jobs: build-and-validate: diff --git a/.github/workflows/sqlite.yml b/.github/workflows/sqlite.yml index d65a66ee..e4867604 100644 --- a/.github/workflows/sqlite.yml +++ b/.github/workflows/sqlite.yml @@ -2,16 +2,11 @@ name: sqlite demo db build on: pull_request_review: types: [submitted] - # TEMP: test on push to branch - push: - branches: - - alistair/more_gh_action_shenanigans jobs: mimic-iv-sqlite: # only run if PR is approved - # TEMP: also run on push - if: github.event_name == 'push' || github.event.review.state == 'approved' + if: github.event.review.state == 'approved' runs-on: ubuntu-latest container: python:3.10 From ad1d23bf34e355b0811a7762698676143565f52d Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Tue, 7 Jul 2026 00:53:58 -0400 Subject: [PATCH 21/26] remove storetime from tie-breaker --- mimic-iv/concepts/firstday/first_day_gcs.sql | 4 ++-- mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql | 2 +- mimic-iv/concepts_postgres/firstday/first_day_gcs.sql | 7 ++----- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/mimic-iv/concepts/firstday/first_day_gcs.sql b/mimic-iv/concepts/firstday/first_day_gcs.sql index bef61c1f..aea6e211 100644 --- a/mimic-iv/concepts/firstday/first_day_gcs.sql +++ b/mimic-iv/concepts/firstday/first_day_gcs.sql @@ -21,13 +21,13 @@ WITH gcs_final AS ( , g.gcs_unable -- This sorts the data by GCS -- rn = 1 is the the lowest total GCS value - -- tie-break on charttime/storetime (unique per stay) so the component columns + -- tie-break on charttime (unique per stay) so the component columns -- are deterministic across SQL engines when multiple measurements -- share the same minimum GCS , ROW_NUMBER() OVER ( PARTITION BY g.stay_id - ORDER BY g.gcs ASC NULLS LAST, g.charttime DESC NULLS LAST, g.storetime DESC NULLS LAST + ORDER BY g.gcs ASC NULLS LAST, g.charttime DESC NULLS LAST ) AS gcs_seq FROM `physionet-data.mimiciv_icu.icustays` ie -- Only get data for the first 24 hours diff --git a/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql b/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql index 13f29b90..19b486b7 100644 --- a/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql +++ b/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql @@ -9,7 +9,7 @@ WITH gcs_final AS ( g.gcs_verbal, g.gcs_eyes, g.gcs_unable, - ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs ASC, g.charttime DESC, g.storetime DESC) AS gcs_seq + ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs ASC, g.charttime DESC) AS gcs_seq FROM mimiciv_icu.icustays AS ie LEFT JOIN mimiciv_derived.gcs AS g ON ie.stay_id = g.stay_id diff --git a/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql b/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql index 2d7424f3..4f42e026 100644 --- a/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql +++ b/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql @@ -9,11 +9,8 @@ WITH gcs_final AS ( g.gcs_motor, g.gcs_verbal, g.gcs_eyes, - g.gcs_unable, /* This sorts the data by GCS */ /* rn = 1 is the the lowest total GCS value */ /* tie-break on charttime/storetime (unique per stay) so the component columns */ /* are deterministic across SQL engines when multiple measurements */ /* share the same minimum GCS */ - ROW_NUMBER() OVER ( - PARTITION BY g.stay_id - ORDER BY g.gcs ASC, g.charttime DESC NULLS LAST, g.storetime DESC NULLS LAST - ) AS gcs_seq + g.gcs_unable, /* This sorts the data by GCS */ /* rn = 1 is the the lowest total GCS value */ /* tie-break on charttime (unique per stay) so the component columns */ /* are deterministic across SQL engines when multiple measurements */ /* share the same minimum GCS */ + ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs ASC, g.charttime DESC NULLS LAST) AS gcs_seq FROM mimiciv_icu.icustays AS ie /* Only get data for the first 24 hours */ LEFT JOIN mimiciv_derived.gcs AS g From 735987f1c19e9791f3a4a09363190c02e0c228f5 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Tue, 7 Jul 2026 08:04:33 -0400 Subject: [PATCH 22/26] fix typo in norepi name, add sepsis3 to skip list --- mimic-iv/concepts/make_concepts.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mimic-iv/concepts/make_concepts.sh b/mimic-iv/concepts/make_concepts.sh index eb5fee25..112bcdc2 100644 --- a/mimic-iv/concepts/make_concepts.sh +++ b/mimic-iv/concepts/make_concepts.sh @@ -63,7 +63,7 @@ do # skip certain tables where order matters skip=0 - for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_eqivalent_dose + for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 do if [[ "${tbl}" == "${skip_table}" ]]; then skip=1 @@ -83,7 +83,7 @@ done echo "Now generating tables which were skipped due to depending on other tables." # generate tables after the above, and in a specific order to ensure dependencies are met -for table_path in firstday/first_day_sofa organfailure/kdigo_stages organfailure/meld medication/vasoactive_agent medication/norepinephrine_equivalent_dose; +for table_path in firstday/first_day_sofa organfailure/kdigo_stages organfailure/meld medication/vasoactive_agent medication/norepinephrine_equivalent_dose sepsis/sepsis3; do table=`echo $table_path | rev | cut -d/ -f1 | rev` From 7214bd358d8164912427b4da827ad4102ed98e9a Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Tue, 7 Jul 2026 22:38:39 -0400 Subject: [PATCH 23/26] delete duplicate queries --- .../comorbidity/first_day_lab.sql | 167 ------------------ .../measurement/kdigo_uo.sql | 2 - 2 files changed, 169 deletions(-) delete mode 100644 mimic-iv/concepts_postgres/comorbidity/first_day_lab.sql delete mode 100644 mimic-iv/concepts_postgres/measurement/kdigo_uo.sql diff --git a/mimic-iv/concepts_postgres/comorbidity/first_day_lab.sql b/mimic-iv/concepts_postgres/comorbidity/first_day_lab.sql deleted file mode 100644 index 54ec040f..00000000 --- a/mimic-iv/concepts_postgres/comorbidity/first_day_lab.sql +++ /dev/null @@ -1,167 +0,0 @@ -DROP TABLE IF EXISTS .first_day_lab; CREATE TABLE .first_day_lab AS -WITH cbc AS -( - SELECT - ie.stay_id - , MIN(hematocrit) as hematocrit_min - , MAX(hematocrit) as hematocrit_max - , MIN(hemoglobin) as hemoglobin_min - , MAX(hemoglobin) as hemoglobin_max - , MIN(platelet) as platelets_min - , MAX(platelet) as platelets_max - , MIN(wbc) as wbc_min - , MAX(wbc) as wbc_max - FROM mimiciv_icu.icustays ie - LEFT JOIN mimiciv_derived.complete_blood_count le - ON le.subject_id = ie.subject_id - AND le.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR) - AND le.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY) - GROUP BY ie.stay_id -) -, chem AS -( - SELECT - ie.stay_id - , MIN(albumin) AS albumin_min, MAX(albumin) AS albumin_max - , MIN(globulin) AS globulin_min, MAX(globulin) AS globulin_max - , MIN(total_protein) AS total_protein_min, MAX(total_protein) AS total_protein_max - , MIN(aniongap) AS aniongap_min, MAX(aniongap) AS aniongap_max - , MIN(bicarbonate) AS bicarbonate_min, MAX(bicarbonate) AS bicarbonate_max - , MIN(bun) AS bun_min, MAX(bun) AS bun_max - , MIN(calcium) AS calcium_min, MAX(calcium) AS calcium_max - , MIN(chloride) AS chloride_min, MAX(chloride) AS chloride_max - , MIN(creatinine) AS creatinine_min, MAX(creatinine) AS creatinine_max - , MIN(glucose) AS glucose_min, MAX(glucose) AS glucose_max - , MIN(sodium) AS sodium_min, MAX(sodium) AS sodium_max - , MIN(potassium) AS potassium_min, MAX(potassium) AS potassium_max - FROM mimiciv_icu.icustays ie - LEFT JOIN mimiciv_derived.chemistry le - ON le.subject_id = ie.subject_id - AND le.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR) - AND le.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY) - GROUP BY ie.stay_id -) -, diff AS -( - SELECT - ie.stay_id - , MIN(basophils_abs) AS basophils_abs_min, MAX(basophils_abs) AS basophils_abs_max - , MIN(eosinophils_abs) AS eosinophils_abs_min, MAX(eosinophils_abs) AS eosinophils_abs_max - , MIN(lymphocytes_abs) AS lymphocytes_abs_min, MAX(lymphocytes_abs) AS lymphocytes_abs_max - , MIN(monocytes_abs) AS monocytes_abs_min, MAX(monocytes_abs) AS monocytes_abs_max - , MIN(neutrophils_abs) AS neutrophils_abs_min, MAX(neutrophils_abs) AS neutrophils_abs_max - , MIN(atypical_lymphocytes) AS atypical_lymphocytes_min, MAX(atypical_lymphocytes) AS atypical_lymphocytes_max - , MIN(bands) AS bands_min, MAX(bands) AS bands_max - , MIN(immature_granulocytes) AS immature_granulocytes_min, MAX(immature_granulocytes) AS immature_granulocytes_max - , MIN(metamyelocytes) AS metamyelocytes_min, MAX(metamyelocytes) AS metamyelocytes_max - , MIN(nrbc) AS nrbc_min, MAX(nrbc) AS nrbc_max - FROM mimiciv_icu.icustays ie - LEFT JOIN mimiciv_derived.blood_differential le - ON le.subject_id = ie.subject_id - AND le.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR) - AND le.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY) - GROUP BY ie.stay_id -) -, coag AS -( - SELECT - ie.stay_id - , MIN(d_dimer) AS d_dimer_min, MAX(d_dimer) AS d_dimer_max - , MIN(fibrinogen) AS fibrinogen_min, MAX(fibrinogen) AS fibrinogen_max - , MIN(thrombin) AS thrombin_min, MAX(thrombin) AS thrombin_max - , MIN(inr) AS inr_min, MAX(inr) AS inr_max - , MIN(pt) AS pt_min, MAX(pt) AS pt_max - , MIN(ptt) AS ptt_min, MAX(ptt) AS ptt_max - FROM mimiciv_icu.icustays ie - LEFT JOIN mimiciv_derived.coagulation le - ON le.subject_id = ie.subject_id - AND le.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR) - AND le.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY) - GROUP BY ie.stay_id -) -, enz AS -( - SELECT - ie.stay_id - - , MIN(alt) AS alt_min, MAX(alt) AS alt_max - , MIN(alp) AS alp_min, MAX(alp) AS alp_max - , MIN(ast) AS ast_min, MAX(ast) AS ast_max - , MIN(amylase) AS amylase_min, MAX(amylase) AS amylase_max - , MIN(bilirubin_total) AS bilirubin_total_min, MAX(bilirubin_total) AS bilirubin_total_max - , MIN(bilirubin_direct) AS bilirubin_direct_min, MAX(bilirubin_direct) AS bilirubin_direct_max - , MIN(bilirubin_indirect) AS bilirubin_indirect_min, MAX(bilirubin_indirect) AS bilirubin_indirect_max - , MIN(ck_cpk) AS ck_cpk_min, MAX(ck_cpk) AS ck_cpk_max - , MIN(ck_mb) AS ck_mb_min, MAX(ck_mb) AS ck_mb_max - , MIN(ggt) AS ggt_min, MAX(ggt) AS ggt_max - , MIN(ld_ldh) AS ld_ldh_min, MAX(ld_ldh) AS ld_ldh_max - FROM mimiciv_icu.icustays ie - LEFT JOIN mimiciv_derived.enzyme le - ON le.subject_id = ie.subject_id - AND le.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR) - AND le.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY) - GROUP BY ie.stay_id -) -SELECT -ie.subject_id -, ie.stay_id --- complete blood count -, hematocrit_min, hematocrit_max -, hemoglobin_min, hemoglobin_max -, platelets_min, platelets_max -, wbc_min, wbc_max --- chemistry -, albumin_min, albumin_max -, globulin_min, globulin_max -, total_protein_min, total_protein_max -, aniongap_min, aniongap_max -, bicarbonate_min, bicarbonate_max -, bun_min, bun_max -, calcium_min, calcium_max -, chloride_min, chloride_max -, creatinine_min, creatinine_max -, glucose_min, glucose_max -, sodium_min, sodium_max -, potassium_min, potassium_max --- blood differential -, basophils_abs_min, basophils_abs_max -, eosinophils_abs_min, eosinophils_abs_max -, lymphocytes_abs_min, lymphocytes_abs_max -, monocytes_abs_min, monocytes_abs_max -, neutrophils_abs_min, neutrophils_abs_max -, atypical_lymphocytes_min, atypical_lymphocytes_max -, bands_min, bands_max -, immature_granulocytes_min, immature_granulocytes_max -, metamyelocytes_min, metamyelocytes_max -, nrbc_min, nrbc_max --- coagulation -, d_dimer_min, d_dimer_max -, fibrinogen_min, fibrinogen_max -, thrombin_min, thrombin_max -, inr_min, inr_max -, pt_min, pt_max -, ptt_min, ptt_max --- enzymes and bilirubin -, alt_min, alt_max -, alp_min, alp_max -, ast_min, ast_max -, amylase_min, amylase_max -, bilirubin_total_min, bilirubin_total_max -, bilirubin_direct_min, bilirubin_direct_max -, bilirubin_indirect_min, bilirubin_indirect_max -, ck_cpk_min, ck_cpk_max -, ck_mb_min, ck_mb_max -, ggt_min, ggt_max -, ld_ldh_min, ld_ldh_max -FROM mimiciv_icu.icustays ie -LEFT JOIN cbc - ON ie.stay_id = cbc.stay_id -LEFT JOIN chem - ON ie.stay_id = chem.stay_id -LEFT JOIN diff - ON ie.stay_id = diff.stay_id -LEFT JOIN coag - ON ie.stay_id = coag.stay_id -LEFT JOIN enz - ON ie.stay_id = enz.stay_id -; diff --git a/mimic-iv/concepts_postgres/measurement/kdigo_uo.sql b/mimic-iv/concepts_postgres/measurement/kdigo_uo.sql deleted file mode 100644 index 645a970d..00000000 --- a/mimic-iv/concepts_postgres/measurement/kdigo_uo.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE IF EXISTS kdigo_uo; CREATE TABLE kdigo_uo AS -DROP TABLE IF EXISTS kdigo_uo; CREATE TABLE kdigo_uo AS From 1482d3dd11db94f5698dfad0ec1e1f9f8528a8ad Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Tue, 7 Jul 2026 22:39:01 -0400 Subject: [PATCH 24/26] remove triggering on push; only trigger on pre-release --- .github/workflows/prerelease_bigquery.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/prerelease_bigquery.yml b/.github/workflows/prerelease_bigquery.yml index afabe84a..01da3995 100644 --- a/.github/workflows/prerelease_bigquery.yml +++ b/.github/workflows/prerelease_bigquery.yml @@ -5,10 +5,6 @@ name: Pre-release concept build on mimiciv_derived on: release: types: [prereleased] - # TEMP: test on push to branch - push: - branches: - - alistair/more_gh_action_shenanigans jobs: build-and-validate: From 3327e48d90dcfe296f65bf4af6d0bb66964f09be Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Tue, 7 Jul 2026 22:39:20 -0400 Subject: [PATCH 25/26] add missing concepts to postgres build --- mimic-iv/concepts_postgres/postgres-make-concepts.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mimic-iv/concepts_postgres/postgres-make-concepts.sql b/mimic-iv/concepts_postgres/postgres-make-concepts.sql index 97e3ea8b..7d2d9bda 100644 --- a/mimic-iv/concepts_postgres/postgres-make-concepts.sql +++ b/mimic-iv/concepts_postgres/postgres-make-concepts.sql @@ -49,6 +49,7 @@ SET search_path TO mimiciv_derived, mimiciv_hosp, mimiciv_icu, mimiciv_ed; -- medication \i medication/acei.sql \i medication/antibiotic.sql +\i medication/arb.sql \i medication/dobutamine.sql \i medication/dopamine.sql \i medication/epinephrine.sql @@ -60,6 +61,7 @@ SET search_path TO mimiciv_derived, mimiciv_hosp, mimiciv_icu, mimiciv_ed; \i medication/vasopressin.sql -- treatment +\i treatment/code_status.sql \i treatment/crrt.sql \i treatment/invasive_line.sql \i treatment/rrt.sql From 321dc69284bc5a5a6034d057c1dff8fbe66a30bd Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Tue, 7 Jul 2026 23:26:01 -0400 Subject: [PATCH 26/26] lint sql --- mimic-iv/concepts/firstday/first_day_rrt.sql | 3 ++- mimic-iv/concepts/measurement/bg.sql | 2 +- mimic-iv/concepts/measurement/rhythm.sql | 4 ++-- mimic-iv/concepts/measurement/urine_output_rate.sql | 12 +++++++++--- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/mimic-iv/concepts/firstday/first_day_rrt.sql b/mimic-iv/concepts/firstday/first_day_rrt.sql index b6e95a74..c051de95 100644 --- a/mimic-iv/concepts/firstday/first_day_rrt.sql +++ b/mimic-iv/concepts/firstday/first_day_rrt.sql @@ -5,7 +5,8 @@ SELECT , ie.stay_id , MAX(dialysis_present) AS dialysis_present , MAX(dialysis_active) AS dialysis_active - , STRING_AGG(DISTINCT dialysis_type, ', ' ORDER BY dialysis_type) AS dialysis_type + , STRING_AGG(DISTINCT dialysis_type, ', ' ORDER BY dialysis_type) + AS dialysis_type FROM `physionet-data.mimiciv_icu.icustays` ie LEFT JOIN `physionet-data.mimiciv_derived.rrt` rrt ON ie.stay_id = rrt.stay_id diff --git a/mimic-iv/concepts/measurement/bg.sql b/mimic-iv/concepts/measurement/bg.sql index 41ffbbab..e3d44eb2 100644 --- a/mimic-iv/concepts/measurement/bg.sql +++ b/mimic-iv/concepts/measurement/bg.sql @@ -200,7 +200,7 @@ SELECT OR pco2 IS NULL THEN NULL WHEN fio2 IS NOT NULL - -- multiple by 100 because fio2 is in a % but should be a fraction + -- multiple by 100 so fio2 goes % -> fraction THEN (fio2 / 100) * (760 - 47) - (pco2 / 0.8) - po2 WHEN fio2_chartevents IS NOT NULL THEN (fio2_chartevents / 100) * (760 - 47) - (pco2 / 0.8) - po2 diff --git a/mimic-iv/concepts/measurement/rhythm.sql b/mimic-iv/concepts/measurement/rhythm.sql index dcf35836..9245edc1 100644 --- a/mimic-iv/concepts/measurement/rhythm.sql +++ b/mimic-iv/concepts/measurement/rhythm.sql @@ -3,8 +3,8 @@ SELECT ce.subject_id , ce.charttime , STRING_AGG( - DISTINCT CASE WHEN itemid = 220048 THEN value ELSE NULL END, - '; ' ORDER BY CASE WHEN itemid = 220048 THEN value ELSE NULL END + DISTINCT CASE WHEN itemid = 220048 THEN value ELSE NULL END + , '; ' ORDER BY CASE WHEN itemid = 220048 THEN value ELSE NULL END ) AS heart_rhythm , MAX(CASE WHEN itemid = 224650 THEN value ELSE NULL END) AS ectopy_type , MAX( diff --git a/mimic-iv/concepts/measurement/urine_output_rate.sql b/mimic-iv/concepts/measurement/urine_output_rate.sql index 99b1311e..01939211 100644 --- a/mimic-iv/concepts/measurement/urine_output_rate.sql +++ b/mimic-iv/concepts/measurement/urine_output_rate.sql @@ -49,7 +49,9 @@ WITH tm AS ( THEN iosum.urineoutput ELSE NULL END) AS urineoutput_6hr , ROUND(CAST( - SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 + SUM( + CASE + WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 THEN iosum.tm_since_last_uo ELSE NULL END) / 60.0 AS NUMERIC ), 6) AS uo_tm_6hr @@ -57,13 +59,17 @@ WITH tm AS ( THEN iosum.urineoutput ELSE NULL END) AS urineoutput_12hr , ROUND(CAST( - SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 + SUM( + CASE + WHEN + DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 THEN iosum.tm_since_last_uo ELSE NULL END) / 60.0 AS NUMERIC ), 6) AS uo_tm_12hr -- 24 hours , SUM(iosum.urineoutput) AS urineoutput_24hr - , ROUND(CAST(SUM(iosum.tm_since_last_uo) / 60.0 AS NUMERIC), 6) AS uo_tm_24hr + , ROUND(CAST(SUM(iosum.tm_since_last_uo) / 60.0 AS NUMERIC), 6) + AS uo_tm_24hr FROM uo_tm io -- this join gives you all UO measurements over a 24 hour period