Fix GH actions and make concepts deterministic

.github/actions/download-demo/action.yml

@@ -10,8 +10,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    # The demo datasets are version-pinned and immutable, so a static key never
-    # goes stale
     - name: Restore demo data from cache
       id: cache
       uses: actions/cache@v4
@@ -27,6 +25,10 @@ runs:
       shell: bash
       working-directory: ${{ inputs.dest }}
       run: |
+        set -euo pipefail
         echo "Downloading MIMIC-IV and MIMIC-IV-ED demo from PhysioNet."
-        wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/
-        wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/
+        # -4 forces IPv4
+        # -c resumes any partially downloaded files on retry.
+        wget_opts="-4 --tries=5 --retry-connrefused --waitretry=10 --timeout=60"
+        wget -r -N -c -q $wget_opts --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/
+        wget -r -N -c -q $wget_opts --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/

.github/actions/setup-duckdb/action.yml

@@ -13,7 +13,11 @@ runs:
     - name: Install DuckDB CLI
       shell: bash
       run: |
-        wget -q "https://github.com/duckdb/duckdb/releases/download/v${{ inputs.version }}/duckdb_cli-linux-amd64.zip"
+        set -euo pipefail
+        url="https://github.com/duckdb/duckdb/releases/download/v${{ inputs.version }}/duckdb_cli-linux-amd64.zip"
+        # -4 forces IPv4, plus retries to survive transient network failures
+        wget -q -4 --tries=5 --retry-connrefused --waitretry=10 --timeout=60 \
+          -O duckdb_cli-linux-amd64.zip "$url"
         unzip -o duckdb_cli-linux-amd64.zip -d /usr/local/bin
         rm duckdb_cli-linux-amd64.zip
         duckdb --version

.github/workflows/build-psql.yml

@@ -14,6 +14,9 @@ jobs:
         image: postgres:17
         env:
           POSTGRES_PASSWORD: postgres
+          # Force byte-order (C) collation so string ordering matches DuckDB's
+          # binary sort.
+          POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C"
         options: >-
           --health-cmd pg_isready
           --health-interval 10s

.github/workflows/concepts-psql.yml

@@ -15,6 +15,9 @@ jobs:
         image: postgres:17
         env:
           POSTGRES_PASSWORD: postgres
+          # Force byte-order (C) collation so string ordering matches DuckDB's
+          # binary sort.
+          POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C"
         options: >-
           --health-cmd pg_isready
           --health-interval 10s

.github/workflows/equivalence.yml

@@ -5,8 +5,8 @@ on:
   workflow_call:
 
 env:
-  # Tables with known cross-engine differences
-  IGNORE_TABLES: "first_day_gcs,suspicion_of_infection,oxygen_delivery,bg,first_day_bg,first_day_bg_art,kdigo_uo,kdigo_stages,urine_output_rate"
+  # If tables are known to be non-equivalent, list them here (comma delimited) to ignore them in the diff.
+  IGNORE_TABLES: ""
 
 jobs:
   diff:
@@ -18,6 +18,9 @@ jobs:
         image: postgres:17
         env:
           POSTGRES_PASSWORD: postgres
+          # Force byte-order (C) collation so string ordering matches DuckDB's
+          # binary sort.
+          POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C"
         options: >-
           --health-cmd pg_isready
           --health-interval 10s

.github/workflows/mysql.yml

@@ -18,9 +18,13 @@ jobs:
 
       - name: Extract demo data to local folder
         run: |
-          mv hosp/*.csv.gz ./
-          mv icu/*.csv.gz ./
-          mv ed/*.csv.gz ./
+          # download-demo provides the gzipped CSVs under hosp/, icu/ and ed/.
+          # Flatten every nested CSV into the working directory, where the mysql
+          # load.sql scripts expect them (e.g. LOAD DATA INFILE 'admissions.csv').
+          # Using find keeps this robust to how deeply the files are nested.
+          # NB: copy (not move) so hosp/, icu/ and ed/ retain their contents
+          # and the cache is preserved.
+          find . -mindepth 2 -name '*.csv.gz' -exec cp -t ./ {} +
           gzip -d *.csv.gz
 
       - name: Start MySQL service

mimic-iv/concepts/demographics/weight_durations.sql

@@ -6,8 +6,7 @@ WITH wt_stg AS (
         , c.charttime
         , CASE WHEN c.itemid = 226512 THEN 'admit'
             ELSE 'daily' END AS weight_type
-        -- TODO: eliminate obvious outliers if there is a reasonable weight
-        , c.valuenum AS weight
+        , ROUND(CAST(c.valuenum AS NUMERIC), 3) AS weight
     FROM `physionet-data.mimiciv_icu.chartevents` c
     WHERE c.valuenum IS NOT NULL
         AND c.itemid IN
@@ -16,6 +15,7 @@ WITH wt_stg AS (
             , 224639 -- Daily Weight
         )
         AND c.valuenum > 0
+        AND c.valuenum < 1500
 )
 
 -- assign ascending row number

mimic-iv/concepts/firstday/first_day_gcs.sql

@@ -21,10 +21,13 @@ WITH gcs_final AS (
         , g.gcs_unable
         -- This sorts the data by GCS
         -- rn = 1 is the the lowest total GCS value
+        -- tie-break on charttime (unique per stay) so the component columns
+        -- are deterministic across SQL engines when multiple measurements
+        -- share the same minimum GCS
         , ROW_NUMBER() OVER
         (
             PARTITION BY g.stay_id
-            ORDER BY g.gcs
+            ORDER BY g.gcs ASC NULLS LAST, g.charttime DESC NULLS LAST
         ) AS gcs_seq
     FROM `physionet-data.mimiciv_icu.icustays` ie
     -- Only get data for the first 24 hours

mimic-iv/concepts/firstday/first_day_rrt.sql

@@ -5,7 +5,8 @@ SELECT
     , ie.stay_id
     , MAX(dialysis_present) AS dialysis_present
     , MAX(dialysis_active) AS dialysis_active
-    , STRING_AGG(DISTINCT dialysis_type, ', ') AS dialysis_type
+    , STRING_AGG(DISTINCT dialysis_type, ', ' ORDER BY dialysis_type)
+        AS dialysis_type
 FROM `physionet-data.mimiciv_icu.icustays` ie
 LEFT JOIN `physionet-data.mimiciv_derived.rrt` rrt
     ON ie.stay_id = rrt.stay_id

mimic-iv/concepts/make_concepts.sh

@@ -63,7 +63,7 @@ do
 
             # skip certain tables where order matters
             skip=0
-            for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_eqivalent_dose
+            for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3
             do
               if [[ "${tbl}" == "${skip_table}" ]]; then
                 skip=1
@@ -83,7 +83,7 @@ done
 
 echo "Now generating tables which were skipped due to depending on other tables."
 # generate tables after the above, and in a specific order to ensure dependencies are met
-for table_path in firstday/first_day_sofa organfailure/kdigo_stages organfailure/meld medication/vasoactive_agent medication/norepinephrine_equivalent_dose;
+for table_path in firstday/first_day_sofa organfailure/kdigo_stages organfailure/meld medication/vasoactive_agent medication/norepinephrine_equivalent_dose sepsis/sepsis3;
 do
   table=`echo $table_path | rev | cut -d/ -f1 | rev`
 

mimic-iv/concepts/measurement/bg.sql

@@ -148,7 +148,7 @@ WITH bg AS (
 , stg2 AS (
     SELECT bg.*
         , ROW_NUMBER() OVER (
-            PARTITION BY bg.subject_id, bg.charttime ORDER BY s1.charttime DESC
+            PARTITION BY bg.specimen_id ORDER BY s1.charttime DESC
         ) AS lastrowspo2
         , s1.spo2
     FROM bg
@@ -165,7 +165,7 @@ WITH bg AS (
 , stg3 AS (
     SELECT bg.*
         , ROW_NUMBER() OVER (
-            PARTITION BY bg.subject_id, bg.charttime ORDER BY s2.charttime DESC
+            PARTITION BY bg.specimen_id ORDER BY s2.charttime DESC
         ) AS lastrowfio2
         , s2.fio2_chartevents
     FROM stg2 bg
@@ -193,18 +193,19 @@ SELECT
     , pco2
     , fio2_chartevents, fio2
     , aado2
-    -- also calculate AADO2
-    , CASE
-        WHEN po2 IS NULL
-            OR pco2 IS NULL
-            THEN NULL
-        WHEN fio2 IS NOT NULL
-            -- multiple by 100 because fio2 is in a % but should be a fraction
-            THEN (fio2 / 100) * (760 - 47) - (pco2 / 0.8) - po2
-        WHEN fio2_chartevents IS NOT NULL
-            THEN (fio2_chartevents / 100) * (760 - 47) - (pco2 / 0.8) - po2
-        ELSE NULL
-    END AS aado2_calc
+    -- also calculate AADO2, rounded to 4 decimal places
+    , ROUND(CAST(
+        CASE
+            WHEN po2 IS NULL
+                OR pco2 IS NULL
+                THEN NULL
+            WHEN fio2 IS NOT NULL
+                -- multiple by 100 so fio2 goes % -> fraction
+                THEN (fio2 / 100) * (760 - 47) - (pco2 / 0.8) - po2
+            WHEN fio2_chartevents IS NOT NULL
+                THEN (fio2_chartevents / 100) * (760 - 47) - (pco2 / 0.8) - po2
+            ELSE NULL
+        END AS NUMERIC), 4) AS aado2_calc
     , CASE
         WHEN po2 IS NULL
             THEN NULL

mimic-iv/concepts/measurement/oxygen_delivery.sql

@@ -34,10 +34,15 @@ WITH ce_stg1 AS (
         , valuenum
         , valueuom
         -- retain only 1 row per charttime
-        -- prioritizing the last documented value
+        -- prioritizing (1) the last documented value
+        -- and (2) the highest value
         -- primarily used to subselect o2 flows
+        -- when storetime ties (e.g. an o2 flow 223834 and a bipap o2 flow
+        -- 227582 documented together), prefer the highest value so the result
+        -- is deterministic across SQL engines
         , ROW_NUMBER() OVER (
-            PARTITION BY subject_id, charttime, itemid ORDER BY storetime DESC
+            PARTITION BY subject_id, charttime, itemid
+            ORDER BY storetime DESC, valuenum DESC
         ) AS rn
     FROM ce_stg1 ce
 )
@@ -62,7 +67,9 @@ WITH ce_stg1 AS (
         , itemid
         , value AS o2_device
         , ROW_NUMBER() OVER (
-            PARTITION BY subject_id, charttime, itemid ORDER BY value
+            PARTITION BY subject_id, charttime, itemid
+            -- sort so the result is deterministic across SQL engines
+            ORDER BY storetime DESC NULLS LAST, value DESC NULLS LAST
         ) AS rn
     FROM `physionet-data.mimiciv_icu.chartevents`
     WHERE itemid = 226732 -- oxygen delivery device(s)

mimic-iv/concepts/measurement/rhythm.sql

@@ -2,7 +2,10 @@
 SELECT
     ce.subject_id
     , ce.charttime
-    , MAX(CASE WHEN itemid = 220048 THEN value ELSE NULL END) AS heart_rhythm
+    , STRING_AGG(
+        DISTINCT CASE WHEN itemid = 220048 THEN value ELSE NULL END
+        , '; ' ORDER BY CASE WHEN itemid = 220048 THEN value ELSE NULL END
+    ) AS heart_rhythm
     , MAX(CASE WHEN itemid = 224650 THEN value ELSE NULL END) AS ectopy_type
     , MAX(
         CASE WHEN itemid = 224651 THEN value ELSE NULL END

mimic-iv/concepts/measurement/urine_output_rate.sql

@@ -48,18 +48,28 @@ WITH tm AS (
         , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5
             THEN iosum.urineoutput
             ELSE NULL END) AS urineoutput_6hr
-        , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5
-            THEN iosum.tm_since_last_uo
-            ELSE NULL END) / 60.0 AS uo_tm_6hr
+        , ROUND(CAST(
+            SUM(
+                CASE
+                    WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5
+                THEN iosum.tm_since_last_uo
+                ELSE NULL END) / 60.0 AS NUMERIC
+        ), 6) AS uo_tm_6hr
         , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11
             THEN iosum.urineoutput
             ELSE NULL END) AS urineoutput_12hr
-        , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11
-            THEN iosum.tm_since_last_uo
-            ELSE NULL END) / 60.0 AS uo_tm_12hr
+        , ROUND(CAST(
+            SUM(
+                CASE
+                    WHEN
+                        DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11
+                THEN iosum.tm_since_last_uo
+                ELSE NULL END) / 60.0 AS NUMERIC
+        ), 6) AS uo_tm_12hr
         -- 24 hours
         , SUM(iosum.urineoutput) AS urineoutput_24hr
-        , SUM(iosum.tm_since_last_uo) / 60.0 AS uo_tm_24hr
+        , ROUND(CAST(SUM(iosum.tm_since_last_uo) / 60.0 AS NUMERIC), 6)
+            AS uo_tm_24hr
 
     FROM uo_tm io
     -- this join gives you all UO measurements over a 24 hour period

mimic-iv/concepts/organfailure/kdigo_uo.sql

@@ -45,26 +45,26 @@ WITH uo_stg1 AS (
 
         -- repeat the summations using the hours_since_previous_row column
         -- this gives us the amount of time the UO was calculated over
-        , SUM(hours_since_previous_row) OVER
+        , ROUND(CAST(SUM(hours_since_previous_row) OVER
         (
             PARTITION BY stay_id
             ORDER BY seconds_since_admit
             RANGE BETWEEN 21600 PRECEDING AND CURRENT ROW
-        ) AS uo_tm_6hr
+        ) AS NUMERIC), 6) AS uo_tm_6hr
 
-        , SUM(hours_since_previous_row) OVER
+        , ROUND(CAST(SUM(hours_since_previous_row) OVER
         (
             PARTITION BY stay_id
             ORDER BY seconds_since_admit
             RANGE BETWEEN 43200 PRECEDING AND CURRENT ROW
-        ) AS uo_tm_12hr
+        ) AS NUMERIC), 6) AS uo_tm_12hr
 
-        , SUM(hours_since_previous_row) OVER
+        , ROUND(CAST(SUM(hours_since_previous_row) OVER
         (
             PARTITION BY stay_id
             ORDER BY seconds_since_admit
             RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW
-        ) AS uo_tm_24hr
+        ) AS NUMERIC), 6) AS uo_tm_24hr
     FROM uo_stg1
 )
 

mimic-iv/concepts/sepsis/suspicion_of_infection.sql

@@ -10,10 +10,12 @@ WITH ab_tbl AS (
         , DATETIME_TRUNC(abx.starttime, DAY) AS antibiotic_date
         , abx.stoptime AS antibiotic_stoptime
         -- create a unique identifier for each patient antibiotic
+        -- hadm_id/stay_id tiebreak keeps the numbering deterministic when
+        -- multiple antibiotics share the same start/stop time and name
         , ROW_NUMBER() OVER
         (
             PARTITION BY subject_id
-            ORDER BY starttime, stoptime, antibiotic
+            ORDER BY starttime, stoptime, antibiotic, hadm_id, stay_id
         ) AS ab_id
     FROM `physionet-data.mimiciv_derived.antibiotic` abx
 )
@@ -57,10 +59,18 @@ WITH ab_tbl AS (
         -- before this abx
         -- this ensures each antibiotic is only matched to a single culture
         -- and consequently we have 1 row per antibiotic
+        -- among cultures at the same time point, prefer a positive culture
+        -- (positiveculture DESC) so that any positive culture yields
+        -- positive_culture = 1, per the suspicion of infection criteria; the
+        -- micro_specimen_id tiebreak makes the selection deterministic
         , ROW_NUMBER() OVER
         (
             PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id
-            ORDER BY me72.chartdate, me72.charttime NULLS LAST
+            ORDER BY
+                me72.chartdate
+                , me72.charttime NULLS LAST
+                , me72.positiveculture DESC
+                , me72.micro_specimen_id
         ) AS micro_seq
     FROM ab_tbl
     -- abx taken after culture, but no more than 72 hours after
@@ -104,10 +114,18 @@ WITH ab_tbl AS (
         -- before this abx
         -- this ensures each antibiotic is only matched to a single culture
         -- and consequently we have 1 row per antibiotic
+        -- among cultures at the same time point, prefer a positive culture
+        -- (positiveculture DESC) so that any positive culture yields
+        -- positive_culture = 1, per the suspicion of infection criteria; the
+        -- micro_specimen_id tiebreak makes the selection deterministic
         , ROW_NUMBER() OVER
         (
             PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id
-            ORDER BY me24.chartdate, me24.charttime NULLS LAST
+            ORDER BY
+                me24.chartdate
+                , me24.charttime NULLS LAST
+                , me24.positiveculture DESC
+                , me24.micro_specimen_id
         ) AS micro_seq
     FROM ab_tbl
     -- culture in subsequent 24 hours

mimic-iv/concepts_duckdb/demographics/weight_durations.sql

@@ -5,10 +5,13 @@ WITH wt_stg AS (
     c.stay_id,
     c.charttime,
     CASE WHEN c.itemid = 226512 THEN 'admit' ELSE 'daily' END AS weight_type,
-    c.valuenum AS weight
+    ROUND(CAST(c.valuenum AS DECIMAL(38, 9)), 3) AS weight
   FROM mimiciv_icu.chartevents AS c
   WHERE
-    NOT c.valuenum IS NULL AND c.itemid IN (226512, 224639) AND c.valuenum > 0
+    NOT c.valuenum IS NULL
+    AND c.itemid IN (226512, 224639)
+    AND c.valuenum > 0
+    AND c.valuenum < 1500
 ), wt_stg1 AS (
   SELECT
     stay_id,

mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql

@@ -9,7 +9,7 @@ WITH gcs_final AS (
     g.gcs_verbal,
     g.gcs_eyes,
     g.gcs_unable,
-    ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs NULLS FIRST) AS gcs_seq
+    ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs ASC, g.charttime DESC) AS gcs_seq
   FROM mimiciv_icu.icustays AS ie
   LEFT JOIN mimiciv_derived.gcs AS g
     ON ie.stay_id = g.stay_id