diff --git a/.github/actions/download-demo/action.yml b/.github/actions/download-demo/action.yml index d4a1a70c2..34525d697 100644 --- a/.github/actions/download-demo/action.yml +++ b/.github/actions/download-demo/action.yml @@ -10,8 +10,6 @@ inputs: runs: using: "composite" steps: - # The demo datasets are version-pinned and immutable, so a static key never - # goes stale - name: Restore demo data from cache id: cache uses: actions/cache@v4 @@ -27,6 +25,10 @@ runs: shell: bash working-directory: ${{ inputs.dest }} run: | + set -euo pipefail echo "Downloading MIMIC-IV and MIMIC-IV-ED demo from PhysioNet." - wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/ - wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/ + # -4 forces IPv4 + # -c resumes any partially downloaded files on retry. + wget_opts="-4 --tries=5 --retry-connrefused --waitretry=10 --timeout=60" + wget -r -N -c -q $wget_opts --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/ + wget -r -N -c -q $wget_opts --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/ diff --git a/.github/actions/setup-duckdb/action.yml b/.github/actions/setup-duckdb/action.yml index c358e449e..91ba9350a 100644 --- a/.github/actions/setup-duckdb/action.yml +++ b/.github/actions/setup-duckdb/action.yml @@ -13,7 +13,11 @@ runs: - name: Install DuckDB CLI shell: bash run: | - wget -q "https://github.com/duckdb/duckdb/releases/download/v${{ inputs.version }}/duckdb_cli-linux-amd64.zip" + set -euo pipefail + url="https://github.com/duckdb/duckdb/releases/download/v${{ inputs.version }}/duckdb_cli-linux-amd64.zip" + # -4 forces IPv4, plus retries to survive transient network failures + wget -q -4 --tries=5 --retry-connrefused --waitretry=10 --timeout=60 \ + -O duckdb_cli-linux-amd64.zip "$url" unzip -o duckdb_cli-linux-amd64.zip -d /usr/local/bin rm duckdb_cli-linux-amd64.zip duckdb --version diff --git a/.github/workflows/build-psql.yml b/.github/workflows/build-psql.yml index cebe1d8c4..2e03054c1 100644 --- a/.github/workflows/build-psql.yml +++ b/.github/workflows/build-psql.yml @@ -14,6 +14,9 @@ jobs: image: postgres:17 env: POSTGRES_PASSWORD: postgres + # Force byte-order (C) collation so string ordering matches DuckDB's + # binary sort. + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" options: >- --health-cmd pg_isready --health-interval 10s diff --git a/.github/workflows/concepts-psql.yml b/.github/workflows/concepts-psql.yml index 320342eb2..5a5c4ec57 100644 --- a/.github/workflows/concepts-psql.yml +++ b/.github/workflows/concepts-psql.yml @@ -15,6 +15,9 @@ jobs: image: postgres:17 env: POSTGRES_PASSWORD: postgres + # Force byte-order (C) collation so string ordering matches DuckDB's + # binary sort. + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" options: >- --health-cmd pg_isready --health-interval 10s diff --git a/.github/workflows/equivalence.yml b/.github/workflows/equivalence.yml index deee28df0..5197aaae5 100644 --- a/.github/workflows/equivalence.yml +++ b/.github/workflows/equivalence.yml @@ -5,8 +5,8 @@ on: workflow_call: env: - # Tables with known cross-engine differences - IGNORE_TABLES: "first_day_gcs,suspicion_of_infection,oxygen_delivery,bg,first_day_bg,first_day_bg_art,kdigo_uo,kdigo_stages,urine_output_rate" + # If tables are known to be non-equivalent, list them here (comma delimited) to ignore them in the diff. + IGNORE_TABLES: "" jobs: diff: @@ -18,6 +18,9 @@ jobs: image: postgres:17 env: POSTGRES_PASSWORD: postgres + # Force byte-order (C) collation so string ordering matches DuckDB's + # binary sort. + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" options: >- --health-cmd pg_isready --health-interval 10s diff --git a/.github/workflows/mysql.yml b/.github/workflows/mysql.yml index 2f1bc695d..76c465ae6 100644 --- a/.github/workflows/mysql.yml +++ b/.github/workflows/mysql.yml @@ -18,9 +18,13 @@ jobs: - name: Extract demo data to local folder run: | - mv hosp/*.csv.gz ./ - mv icu/*.csv.gz ./ - mv ed/*.csv.gz ./ + # download-demo provides the gzipped CSVs under hosp/, icu/ and ed/. + # Flatten every nested CSV into the working directory, where the mysql + # load.sql scripts expect them (e.g. LOAD DATA INFILE 'admissions.csv'). + # Using find keeps this robust to how deeply the files are nested. + # NB: copy (not move) so hosp/, icu/ and ed/ retain their contents + # and the cache is preserved. + find . -mindepth 2 -name '*.csv.gz' -exec cp -t ./ {} + gzip -d *.csv.gz - name: Start MySQL service diff --git a/mimic-iv/concepts/demographics/weight_durations.sql b/mimic-iv/concepts/demographics/weight_durations.sql index de1b5cdfb..a28bb4f6c 100644 --- a/mimic-iv/concepts/demographics/weight_durations.sql +++ b/mimic-iv/concepts/demographics/weight_durations.sql @@ -6,8 +6,7 @@ WITH wt_stg AS ( , c.charttime , CASE WHEN c.itemid = 226512 THEN 'admit' ELSE 'daily' END AS weight_type - -- TODO: eliminate obvious outliers if there is a reasonable weight - , c.valuenum AS weight + , ROUND(CAST(c.valuenum AS NUMERIC), 3) AS weight FROM `physionet-data.mimiciv_icu.chartevents` c WHERE c.valuenum IS NOT NULL AND c.itemid IN @@ -16,6 +15,7 @@ WITH wt_stg AS ( , 224639 -- Daily Weight ) AND c.valuenum > 0 + AND c.valuenum < 1500 ) -- assign ascending row number diff --git a/mimic-iv/concepts/firstday/first_day_gcs.sql b/mimic-iv/concepts/firstday/first_day_gcs.sql index b74698520..aea6e2117 100644 --- a/mimic-iv/concepts/firstday/first_day_gcs.sql +++ b/mimic-iv/concepts/firstday/first_day_gcs.sql @@ -21,10 +21,13 @@ WITH gcs_final AS ( , g.gcs_unable -- This sorts the data by GCS -- rn = 1 is the the lowest total GCS value + -- tie-break on charttime (unique per stay) so the component columns + -- are deterministic across SQL engines when multiple measurements + -- share the same minimum GCS , ROW_NUMBER() OVER ( PARTITION BY g.stay_id - ORDER BY g.gcs + ORDER BY g.gcs ASC NULLS LAST, g.charttime DESC NULLS LAST ) AS gcs_seq FROM `physionet-data.mimiciv_icu.icustays` ie -- Only get data for the first 24 hours diff --git a/mimic-iv/concepts/firstday/first_day_rrt.sql b/mimic-iv/concepts/firstday/first_day_rrt.sql index 2f99f12ec..c051de956 100644 --- a/mimic-iv/concepts/firstday/first_day_rrt.sql +++ b/mimic-iv/concepts/firstday/first_day_rrt.sql @@ -5,7 +5,8 @@ SELECT , ie.stay_id , MAX(dialysis_present) AS dialysis_present , MAX(dialysis_active) AS dialysis_active - , STRING_AGG(DISTINCT dialysis_type, ', ') AS dialysis_type + , STRING_AGG(DISTINCT dialysis_type, ', ' ORDER BY dialysis_type) + AS dialysis_type FROM `physionet-data.mimiciv_icu.icustays` ie LEFT JOIN `physionet-data.mimiciv_derived.rrt` rrt ON ie.stay_id = rrt.stay_id diff --git a/mimic-iv/concepts/make_concepts.sh b/mimic-iv/concepts/make_concepts.sh index eb5fee25a..112bcdc22 100644 --- a/mimic-iv/concepts/make_concepts.sh +++ b/mimic-iv/concepts/make_concepts.sh @@ -63,7 +63,7 @@ do # skip certain tables where order matters skip=0 - for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_eqivalent_dose + for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 do if [[ "${tbl}" == "${skip_table}" ]]; then skip=1 @@ -83,7 +83,7 @@ done echo "Now generating tables which were skipped due to depending on other tables." # generate tables after the above, and in a specific order to ensure dependencies are met -for table_path in firstday/first_day_sofa organfailure/kdigo_stages organfailure/meld medication/vasoactive_agent medication/norepinephrine_equivalent_dose; +for table_path in firstday/first_day_sofa organfailure/kdigo_stages organfailure/meld medication/vasoactive_agent medication/norepinephrine_equivalent_dose sepsis/sepsis3; do table=`echo $table_path | rev | cut -d/ -f1 | rev` diff --git a/mimic-iv/concepts/measurement/bg.sql b/mimic-iv/concepts/measurement/bg.sql index c464acd43..e3d44eb21 100644 --- a/mimic-iv/concepts/measurement/bg.sql +++ b/mimic-iv/concepts/measurement/bg.sql @@ -148,7 +148,7 @@ WITH bg AS ( , stg2 AS ( SELECT bg.* , ROW_NUMBER() OVER ( - PARTITION BY bg.subject_id, bg.charttime ORDER BY s1.charttime DESC + PARTITION BY bg.specimen_id ORDER BY s1.charttime DESC ) AS lastrowspo2 , s1.spo2 FROM bg @@ -165,7 +165,7 @@ WITH bg AS ( , stg3 AS ( SELECT bg.* , ROW_NUMBER() OVER ( - PARTITION BY bg.subject_id, bg.charttime ORDER BY s2.charttime DESC + PARTITION BY bg.specimen_id ORDER BY s2.charttime DESC ) AS lastrowfio2 , s2.fio2_chartevents FROM stg2 bg @@ -193,18 +193,19 @@ SELECT , pco2 , fio2_chartevents, fio2 , aado2 - -- also calculate AADO2 - , CASE - WHEN po2 IS NULL - OR pco2 IS NULL - THEN NULL - WHEN fio2 IS NOT NULL - -- multiple by 100 because fio2 is in a % but should be a fraction - THEN (fio2 / 100) * (760 - 47) - (pco2 / 0.8) - po2 - WHEN fio2_chartevents IS NOT NULL - THEN (fio2_chartevents / 100) * (760 - 47) - (pco2 / 0.8) - po2 - ELSE NULL - END AS aado2_calc + -- also calculate AADO2, rounded to 4 decimal places + , ROUND(CAST( + CASE + WHEN po2 IS NULL + OR pco2 IS NULL + THEN NULL + WHEN fio2 IS NOT NULL + -- multiple by 100 so fio2 goes % -> fraction + THEN (fio2 / 100) * (760 - 47) - (pco2 / 0.8) - po2 + WHEN fio2_chartevents IS NOT NULL + THEN (fio2_chartevents / 100) * (760 - 47) - (pco2 / 0.8) - po2 + ELSE NULL + END AS NUMERIC), 4) AS aado2_calc , CASE WHEN po2 IS NULL THEN NULL diff --git a/mimic-iv/concepts/measurement/oxygen_delivery.sql b/mimic-iv/concepts/measurement/oxygen_delivery.sql index fc2bbf7fc..e76f63036 100644 --- a/mimic-iv/concepts/measurement/oxygen_delivery.sql +++ b/mimic-iv/concepts/measurement/oxygen_delivery.sql @@ -34,10 +34,15 @@ WITH ce_stg1 AS ( , valuenum , valueuom -- retain only 1 row per charttime - -- prioritizing the last documented value + -- prioritizing (1) the last documented value + -- and (2) the highest value -- primarily used to subselect o2 flows + -- when storetime ties (e.g. an o2 flow 223834 and a bipap o2 flow + -- 227582 documented together), prefer the highest value so the result + -- is deterministic across SQL engines , ROW_NUMBER() OVER ( - PARTITION BY subject_id, charttime, itemid ORDER BY storetime DESC + PARTITION BY subject_id, charttime, itemid + ORDER BY storetime DESC, valuenum DESC ) AS rn FROM ce_stg1 ce ) @@ -62,7 +67,9 @@ WITH ce_stg1 AS ( , itemid , value AS o2_device , ROW_NUMBER() OVER ( - PARTITION BY subject_id, charttime, itemid ORDER BY value + PARTITION BY subject_id, charttime, itemid + -- sort so the result is deterministic across SQL engines + ORDER BY storetime DESC NULLS LAST, value DESC NULLS LAST ) AS rn FROM `physionet-data.mimiciv_icu.chartevents` WHERE itemid = 226732 -- oxygen delivery device(s) diff --git a/mimic-iv/concepts/measurement/rhythm.sql b/mimic-iv/concepts/measurement/rhythm.sql index 20b9d8b5f..9245edc1f 100644 --- a/mimic-iv/concepts/measurement/rhythm.sql +++ b/mimic-iv/concepts/measurement/rhythm.sql @@ -2,7 +2,10 @@ SELECT ce.subject_id , ce.charttime - , MAX(CASE WHEN itemid = 220048 THEN value ELSE NULL END) AS heart_rhythm + , STRING_AGG( + DISTINCT CASE WHEN itemid = 220048 THEN value ELSE NULL END + , '; ' ORDER BY CASE WHEN itemid = 220048 THEN value ELSE NULL END + ) AS heart_rhythm , MAX(CASE WHEN itemid = 224650 THEN value ELSE NULL END) AS ectopy_type , MAX( CASE WHEN itemid = 224651 THEN value ELSE NULL END diff --git a/mimic-iv/concepts/measurement/urine_output_rate.sql b/mimic-iv/concepts/measurement/urine_output_rate.sql index 0a9a2264e..019392118 100644 --- a/mimic-iv/concepts/measurement/urine_output_rate.sql +++ b/mimic-iv/concepts/measurement/urine_output_rate.sql @@ -48,18 +48,28 @@ WITH tm AS ( , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_6hr - , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 - THEN iosum.tm_since_last_uo - ELSE NULL END) / 60.0 AS uo_tm_6hr + , ROUND(CAST( + SUM( + CASE + WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 + THEN iosum.tm_since_last_uo + ELSE NULL END) / 60.0 AS NUMERIC + ), 6) AS uo_tm_6hr , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_12hr - , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 - THEN iosum.tm_since_last_uo - ELSE NULL END) / 60.0 AS uo_tm_12hr + , ROUND(CAST( + SUM( + CASE + WHEN + DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 + THEN iosum.tm_since_last_uo + ELSE NULL END) / 60.0 AS NUMERIC + ), 6) AS uo_tm_12hr -- 24 hours , SUM(iosum.urineoutput) AS urineoutput_24hr - , SUM(iosum.tm_since_last_uo) / 60.0 AS uo_tm_24hr + , ROUND(CAST(SUM(iosum.tm_since_last_uo) / 60.0 AS NUMERIC), 6) + AS uo_tm_24hr FROM uo_tm io -- this join gives you all UO measurements over a 24 hour period diff --git a/mimic-iv/concepts/organfailure/kdigo_uo.sql b/mimic-iv/concepts/organfailure/kdigo_uo.sql index 32119dc9b..c85d4223c 100644 --- a/mimic-iv/concepts/organfailure/kdigo_uo.sql +++ b/mimic-iv/concepts/organfailure/kdigo_uo.sql @@ -45,26 +45,26 @@ WITH uo_stg1 AS ( -- repeat the summations using the hours_since_previous_row column -- this gives us the amount of time the UO was calculated over - , SUM(hours_since_previous_row) OVER + , ROUND(CAST(SUM(hours_since_previous_row) OVER ( PARTITION BY stay_id ORDER BY seconds_since_admit RANGE BETWEEN 21600 PRECEDING AND CURRENT ROW - ) AS uo_tm_6hr + ) AS NUMERIC), 6) AS uo_tm_6hr - , SUM(hours_since_previous_row) OVER + , ROUND(CAST(SUM(hours_since_previous_row) OVER ( PARTITION BY stay_id ORDER BY seconds_since_admit RANGE BETWEEN 43200 PRECEDING AND CURRENT ROW - ) AS uo_tm_12hr + ) AS NUMERIC), 6) AS uo_tm_12hr - , SUM(hours_since_previous_row) OVER + , ROUND(CAST(SUM(hours_since_previous_row) OVER ( PARTITION BY stay_id ORDER BY seconds_since_admit RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW - ) AS uo_tm_24hr + ) AS NUMERIC), 6) AS uo_tm_24hr FROM uo_stg1 ) diff --git a/mimic-iv/concepts/sepsis/suspicion_of_infection.sql b/mimic-iv/concepts/sepsis/suspicion_of_infection.sql index 5b7db295f..2fb271b42 100644 --- a/mimic-iv/concepts/sepsis/suspicion_of_infection.sql +++ b/mimic-iv/concepts/sepsis/suspicion_of_infection.sql @@ -10,10 +10,12 @@ WITH ab_tbl AS ( , DATETIME_TRUNC(abx.starttime, DAY) AS antibiotic_date , abx.stoptime AS antibiotic_stoptime -- create a unique identifier for each patient antibiotic + -- hadm_id/stay_id tiebreak keeps the numbering deterministic when + -- multiple antibiotics share the same start/stop time and name , ROW_NUMBER() OVER ( PARTITION BY subject_id - ORDER BY starttime, stoptime, antibiotic + ORDER BY starttime, stoptime, antibiotic, hadm_id, stay_id ) AS ab_id FROM `physionet-data.mimiciv_derived.antibiotic` abx ) @@ -57,10 +59,18 @@ WITH ab_tbl AS ( -- before this abx -- this ensures each antibiotic is only matched to a single culture -- and consequently we have 1 row per antibiotic + -- among cultures at the same time point, prefer a positive culture + -- (positiveculture DESC) so that any positive culture yields + -- positive_culture = 1, per the suspicion of infection criteria; the + -- micro_specimen_id tiebreak makes the selection deterministic , ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me72.chartdate, me72.charttime NULLS LAST + ORDER BY + me72.chartdate + , me72.charttime NULLS LAST + , me72.positiveculture DESC + , me72.micro_specimen_id ) AS micro_seq FROM ab_tbl -- abx taken after culture, but no more than 72 hours after @@ -104,10 +114,18 @@ WITH ab_tbl AS ( -- before this abx -- this ensures each antibiotic is only matched to a single culture -- and consequently we have 1 row per antibiotic + -- among cultures at the same time point, prefer a positive culture + -- (positiveculture DESC) so that any positive culture yields + -- positive_culture = 1, per the suspicion of infection criteria; the + -- micro_specimen_id tiebreak makes the selection deterministic , ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me24.chartdate, me24.charttime NULLS LAST + ORDER BY + me24.chartdate + , me24.charttime NULLS LAST + , me24.positiveculture DESC + , me24.micro_specimen_id ) AS micro_seq FROM ab_tbl -- culture in subsequent 24 hours diff --git a/mimic-iv/concepts_duckdb/demographics/weight_durations.sql b/mimic-iv/concepts_duckdb/demographics/weight_durations.sql index b30ea0095..f47d63d0f 100644 --- a/mimic-iv/concepts_duckdb/demographics/weight_durations.sql +++ b/mimic-iv/concepts_duckdb/demographics/weight_durations.sql @@ -5,10 +5,13 @@ WITH wt_stg AS ( c.stay_id, c.charttime, CASE WHEN c.itemid = 226512 THEN 'admit' ELSE 'daily' END AS weight_type, - c.valuenum AS weight + ROUND(CAST(c.valuenum AS DECIMAL(38, 9)), 3) AS weight FROM mimiciv_icu.chartevents AS c WHERE - NOT c.valuenum IS NULL AND c.itemid IN (226512, 224639) AND c.valuenum > 0 + NOT c.valuenum IS NULL + AND c.itemid IN (226512, 224639) + AND c.valuenum > 0 + AND c.valuenum < 1500 ), wt_stg1 AS ( SELECT stay_id, diff --git a/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql b/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql index d11e2f9a8..19b486b74 100644 --- a/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql +++ b/mimic-iv/concepts_duckdb/firstday/first_day_gcs.sql @@ -9,7 +9,7 @@ WITH gcs_final AS ( g.gcs_verbal, g.gcs_eyes, g.gcs_unable, - ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs NULLS FIRST) AS gcs_seq + ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs ASC, g.charttime DESC) AS gcs_seq FROM mimiciv_icu.icustays AS ie LEFT JOIN mimiciv_derived.gcs AS g ON ie.stay_id = g.stay_id diff --git a/mimic-iv/concepts_duckdb/firstday/first_day_rrt.sql b/mimic-iv/concepts_duckdb/firstday/first_day_rrt.sql index de2901e07..3c7105808 100644 --- a/mimic-iv/concepts_duckdb/firstday/first_day_rrt.sql +++ b/mimic-iv/concepts_duckdb/firstday/first_day_rrt.sql @@ -5,7 +5,9 @@ SELECT ie.stay_id, MAX(dialysis_present) AS dialysis_present, MAX(dialysis_active) AS dialysis_active, - LISTAGG(DISTINCT dialysis_type, ', ') AS dialysis_type + LISTAGG(DISTINCT dialysis_type, ', ' + ORDER BY + dialysis_type NULLS FIRST) AS dialysis_type FROM mimiciv_icu.icustays AS ie LEFT JOIN mimiciv_derived.rrt AS rrt ON ie.stay_id = rrt.stay_id diff --git a/mimic-iv/concepts_duckdb/measurement/bg.sql b/mimic-iv/concepts_duckdb/measurement/bg.sql index 01e87e5ce..79286421d 100644 --- a/mimic-iv/concepts_duckdb/measurement/bg.sql +++ b/mimic-iv/concepts_duckdb/measurement/bg.sql @@ -110,7 +110,7 @@ WITH bg AS ( ), stg2 AS ( SELECT bg.*, - ROW_NUMBER() OVER (PARTITION BY bg.subject_id, bg.charttime ORDER BY s1.charttime DESC) AS lastrowspo2, + ROW_NUMBER() OVER (PARTITION BY bg.specimen_id ORDER BY s1.charttime DESC) AS lastrowspo2, s1.spo2 FROM bg LEFT JOIN stg_spo2 AS s1 @@ -121,7 +121,7 @@ WITH bg AS ( ), stg3 AS ( SELECT bg.*, - ROW_NUMBER() OVER (PARTITION BY bg.subject_id, bg.charttime ORDER BY s2.charttime DESC) AS lastrowfio2, + ROW_NUMBER() OVER (PARTITION BY bg.specimen_id ORDER BY s2.charttime DESC) AS lastrowfio2, s2.fio2_chartevents FROM stg2 AS bg LEFT JOIN stg_fio2 AS s2 @@ -143,27 +143,30 @@ SELECT fio2_chartevents, fio2, aado2, - CASE - WHEN po2 IS NULL OR pco2 IS NULL - THEN NULL - WHEN NOT fio2 IS NULL - THEN ( - fio2 / 100 - ) * ( - 760 - 47 - ) - ( - pco2 / 0.8 - ) - po2 - WHEN NOT fio2_chartevents IS NULL - THEN ( - fio2_chartevents / 100 - ) * ( - 760 - 47 - ) - ( - pco2 / 0.8 - ) - po2 - ELSE NULL - END AS aado2_calc, + ROUND( + CAST(CASE + WHEN po2 IS NULL OR pco2 IS NULL + THEN NULL + WHEN NOT fio2 IS NULL + THEN ( + fio2 / 100 + ) * ( + 760 - 47 + ) - ( + pco2 / 0.8 + ) - po2 + WHEN NOT fio2_chartevents IS NULL + THEN ( + fio2_chartevents / 100 + ) * ( + 760 - 47 + ) - ( + pco2 / 0.8 + ) - po2 + ELSE NULL + END AS DECIMAL(38, 9)), + 4 + ) AS aado2_calc, CASE WHEN po2 IS NULL THEN NULL diff --git a/mimic-iv/concepts_duckdb/measurement/oxygen_delivery.sql b/mimic-iv/concepts_duckdb/measurement/oxygen_delivery.sql index e3a5b955f..0bfef488c 100644 --- a/mimic-iv/concepts_duckdb/measurement/oxygen_delivery.sql +++ b/mimic-iv/concepts_duckdb/measurement/oxygen_delivery.sql @@ -22,7 +22,7 @@ WITH ce_stg1 AS ( value, valuenum, valueuom, - ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY storetime DESC) AS rn + ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY storetime DESC, valuenum DESC) AS rn FROM ce_stg1 AS ce ), o2 AS ( SELECT @@ -31,7 +31,7 @@ WITH ce_stg1 AS ( charttime, itemid, value AS o2_device, - ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY value NULLS FIRST) AS rn + ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY storetime DESC, value DESC) AS rn FROM mimiciv_icu.chartevents WHERE itemid = 226732 diff --git a/mimic-iv/concepts_duckdb/measurement/rhythm.sql b/mimic-iv/concepts_duckdb/measurement/rhythm.sql index 9a570a977..1907a9e21 100644 --- a/mimic-iv/concepts_duckdb/measurement/rhythm.sql +++ b/mimic-iv/concepts_duckdb/measurement/rhythm.sql @@ -3,7 +3,11 @@ DROP TABLE IF EXISTS mimiciv_derived.rhythm; CREATE TABLE mimiciv_derived.rhythm SELECT ce.subject_id, ce.charttime, - MAX(CASE WHEN itemid = 220048 THEN value ELSE NULL END) AS heart_rhythm, + LISTAGG( + DISTINCT CASE WHEN itemid = 220048 THEN value ELSE NULL END, '; ' + ORDER BY + CASE WHEN itemid = 220048 THEN value ELSE NULL END NULLS FIRST + ) AS heart_rhythm, MAX(CASE WHEN itemid = 224650 THEN value ELSE NULL END) AS ectopy_type, MAX(CASE WHEN itemid = 224651 THEN value ELSE NULL END) AS ectopy_frequency, MAX(CASE WHEN itemid = 226479 THEN value ELSE NULL END) AS ectopy_type_secondary, diff --git a/mimic-iv/concepts_duckdb/measurement/urine_output_rate.sql b/mimic-iv/concepts_duckdb/measurement/urine_output_rate.sql index 21f9c491d..f39e4b417 100644 --- a/mimic-iv/concepts_duckdb/measurement/urine_output_rate.sql +++ b/mimic-iv/concepts_duckdb/measurement/urine_output_rate.sql @@ -39,13 +39,16 @@ WITH tm AS ( ELSE NULL END ) AS urineoutput_6hr, - SUM( - CASE - WHEN DATE_DIFF('HOUR', iosum.charttime, io.charttime) <= 5 - THEN iosum.tm_since_last_uo - ELSE NULL - END - ) / 60.0 AS uo_tm_6hr, + ROUND( + CAST(SUM( + CASE + WHEN DATE_DIFF('HOUR', iosum.charttime, io.charttime) <= 5 + THEN iosum.tm_since_last_uo + ELSE NULL + END + ) / 60.0 AS DECIMAL(38, 9)), + 6 + ) AS uo_tm_6hr, SUM( CASE WHEN DATE_DIFF('HOUR', iosum.charttime, io.charttime) <= 11 @@ -53,15 +56,18 @@ WITH tm AS ( ELSE NULL END ) AS urineoutput_12hr, - SUM( - CASE - WHEN DATE_DIFF('HOUR', iosum.charttime, io.charttime) <= 11 - THEN iosum.tm_since_last_uo - ELSE NULL - END - ) / 60.0 AS uo_tm_12hr, + ROUND( + CAST(SUM( + CASE + WHEN DATE_DIFF('HOUR', iosum.charttime, io.charttime) <= 11 + THEN iosum.tm_since_last_uo + ELSE NULL + END + ) / 60.0 AS DECIMAL(38, 9)), + 6 + ) AS uo_tm_12hr, SUM(iosum.urineoutput) AS urineoutput_24hr, - SUM(iosum.tm_since_last_uo) / 60.0 AS uo_tm_24hr + ROUND(CAST(SUM(iosum.tm_since_last_uo) / 60.0 AS DECIMAL(38, 9)), 6) AS uo_tm_24hr FROM uo_tm AS io LEFT JOIN uo_tm AS iosum ON io.stay_id = iosum.stay_id diff --git a/mimic-iv/concepts_duckdb/organfailure/kdigo_uo.sql b/mimic-iv/concepts_duckdb/organfailure/kdigo_uo.sql index 641f536e8..f2d157903 100644 --- a/mimic-iv/concepts_duckdb/organfailure/kdigo_uo.sql +++ b/mimic-iv/concepts_duckdb/organfailure/kdigo_uo.sql @@ -38,20 +38,29 @@ WITH uo_stg1 AS ( ORDER BY seconds_since_admit NULLS FIRST RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW ) AS urineoutput_24hr, - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 21600 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 21600 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_6hr, - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 43200 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 43200 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_12hr, - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_24hr FROM uo_stg1 ) diff --git a/mimic-iv/concepts_duckdb/sepsis/suspicion_of_infection.sql b/mimic-iv/concepts_duckdb/sepsis/suspicion_of_infection.sql index e2bce86e2..f18ccd8f9 100644 --- a/mimic-iv/concepts_duckdb/sepsis/suspicion_of_infection.sql +++ b/mimic-iv/concepts_duckdb/sepsis/suspicion_of_infection.sql @@ -11,7 +11,7 @@ WITH ab_tbl AS ( abx.stoptime AS antibiotic_stoptime, ROW_NUMBER() OVER ( PARTITION BY subject_id - ORDER BY starttime NULLS FIRST, stoptime NULLS FIRST, antibiotic NULLS FIRST + ORDER BY starttime NULLS FIRST, stoptime NULLS FIRST, antibiotic NULLS FIRST, hadm_id NULLS FIRST, stay_id NULLS FIRST ) AS ab_id FROM mimiciv_derived.antibiotic AS abx ), me AS ( @@ -44,7 +44,7 @@ WITH ab_tbl AS ( me72.spec_type_desc AS last72_specimen, ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me72.chartdate NULLS FIRST, me72.charttime + ORDER BY me72.chartdate NULLS FIRST, me72.charttime, me72.positiveculture DESC, me72.micro_specimen_id NULLS FIRST ) AS micro_seq FROM ab_tbl LEFT JOIN me AS me72 @@ -73,7 +73,7 @@ WITH ab_tbl AS ( me24.spec_type_desc AS next24_specimen, ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me24.chartdate NULLS FIRST, me24.charttime + ORDER BY me24.chartdate NULLS FIRST, me24.charttime, me24.positiveculture DESC, me24.micro_specimen_id NULLS FIRST ) AS micro_seq FROM ab_tbl LEFT JOIN me AS me24 diff --git a/mimic-iv/concepts_postgres/comorbidity/first_day_lab.sql b/mimic-iv/concepts_postgres/comorbidity/first_day_lab.sql deleted file mode 100644 index 54ec040ff..000000000 --- a/mimic-iv/concepts_postgres/comorbidity/first_day_lab.sql +++ /dev/null @@ -1,167 +0,0 @@ -DROP TABLE IF EXISTS .first_day_lab; CREATE TABLE .first_day_lab AS -WITH cbc AS -( - SELECT - ie.stay_id - , MIN(hematocrit) as hematocrit_min - , MAX(hematocrit) as hematocrit_max - , MIN(hemoglobin) as hemoglobin_min - , MAX(hemoglobin) as hemoglobin_max - , MIN(platelet) as platelets_min - , MAX(platelet) as platelets_max - , MIN(wbc) as wbc_min - , MAX(wbc) as wbc_max - FROM mimiciv_icu.icustays ie - LEFT JOIN mimiciv_derived.complete_blood_count le - ON le.subject_id = ie.subject_id - AND le.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR) - AND le.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY) - GROUP BY ie.stay_id -) -, chem AS -( - SELECT - ie.stay_id - , MIN(albumin) AS albumin_min, MAX(albumin) AS albumin_max - , MIN(globulin) AS globulin_min, MAX(globulin) AS globulin_max - , MIN(total_protein) AS total_protein_min, MAX(total_protein) AS total_protein_max - , MIN(aniongap) AS aniongap_min, MAX(aniongap) AS aniongap_max - , MIN(bicarbonate) AS bicarbonate_min, MAX(bicarbonate) AS bicarbonate_max - , MIN(bun) AS bun_min, MAX(bun) AS bun_max - , MIN(calcium) AS calcium_min, MAX(calcium) AS calcium_max - , MIN(chloride) AS chloride_min, MAX(chloride) AS chloride_max - , MIN(creatinine) AS creatinine_min, MAX(creatinine) AS creatinine_max - , MIN(glucose) AS glucose_min, MAX(glucose) AS glucose_max - , MIN(sodium) AS sodium_min, MAX(sodium) AS sodium_max - , MIN(potassium) AS potassium_min, MAX(potassium) AS potassium_max - FROM mimiciv_icu.icustays ie - LEFT JOIN mimiciv_derived.chemistry le - ON le.subject_id = ie.subject_id - AND le.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR) - AND le.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY) - GROUP BY ie.stay_id -) -, diff AS -( - SELECT - ie.stay_id - , MIN(basophils_abs) AS basophils_abs_min, MAX(basophils_abs) AS basophils_abs_max - , MIN(eosinophils_abs) AS eosinophils_abs_min, MAX(eosinophils_abs) AS eosinophils_abs_max - , MIN(lymphocytes_abs) AS lymphocytes_abs_min, MAX(lymphocytes_abs) AS lymphocytes_abs_max - , MIN(monocytes_abs) AS monocytes_abs_min, MAX(monocytes_abs) AS monocytes_abs_max - , MIN(neutrophils_abs) AS neutrophils_abs_min, MAX(neutrophils_abs) AS neutrophils_abs_max - , MIN(atypical_lymphocytes) AS atypical_lymphocytes_min, MAX(atypical_lymphocytes) AS atypical_lymphocytes_max - , MIN(bands) AS bands_min, MAX(bands) AS bands_max - , MIN(immature_granulocytes) AS immature_granulocytes_min, MAX(immature_granulocytes) AS immature_granulocytes_max - , MIN(metamyelocytes) AS metamyelocytes_min, MAX(metamyelocytes) AS metamyelocytes_max - , MIN(nrbc) AS nrbc_min, MAX(nrbc) AS nrbc_max - FROM mimiciv_icu.icustays ie - LEFT JOIN mimiciv_derived.blood_differential le - ON le.subject_id = ie.subject_id - AND le.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR) - AND le.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY) - GROUP BY ie.stay_id -) -, coag AS -( - SELECT - ie.stay_id - , MIN(d_dimer) AS d_dimer_min, MAX(d_dimer) AS d_dimer_max - , MIN(fibrinogen) AS fibrinogen_min, MAX(fibrinogen) AS fibrinogen_max - , MIN(thrombin) AS thrombin_min, MAX(thrombin) AS thrombin_max - , MIN(inr) AS inr_min, MAX(inr) AS inr_max - , MIN(pt) AS pt_min, MAX(pt) AS pt_max - , MIN(ptt) AS ptt_min, MAX(ptt) AS ptt_max - FROM mimiciv_icu.icustays ie - LEFT JOIN mimiciv_derived.coagulation le - ON le.subject_id = ie.subject_id - AND le.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR) - AND le.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY) - GROUP BY ie.stay_id -) -, enz AS -( - SELECT - ie.stay_id - - , MIN(alt) AS alt_min, MAX(alt) AS alt_max - , MIN(alp) AS alp_min, MAX(alp) AS alp_max - , MIN(ast) AS ast_min, MAX(ast) AS ast_max - , MIN(amylase) AS amylase_min, MAX(amylase) AS amylase_max - , MIN(bilirubin_total) AS bilirubin_total_min, MAX(bilirubin_total) AS bilirubin_total_max - , MIN(bilirubin_direct) AS bilirubin_direct_min, MAX(bilirubin_direct) AS bilirubin_direct_max - , MIN(bilirubin_indirect) AS bilirubin_indirect_min, MAX(bilirubin_indirect) AS bilirubin_indirect_max - , MIN(ck_cpk) AS ck_cpk_min, MAX(ck_cpk) AS ck_cpk_max - , MIN(ck_mb) AS ck_mb_min, MAX(ck_mb) AS ck_mb_max - , MIN(ggt) AS ggt_min, MAX(ggt) AS ggt_max - , MIN(ld_ldh) AS ld_ldh_min, MAX(ld_ldh) AS ld_ldh_max - FROM mimiciv_icu.icustays ie - LEFT JOIN mimiciv_derived.enzyme le - ON le.subject_id = ie.subject_id - AND le.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR) - AND le.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY) - GROUP BY ie.stay_id -) -SELECT -ie.subject_id -, ie.stay_id --- complete blood count -, hematocrit_min, hematocrit_max -, hemoglobin_min, hemoglobin_max -, platelets_min, platelets_max -, wbc_min, wbc_max --- chemistry -, albumin_min, albumin_max -, globulin_min, globulin_max -, total_protein_min, total_protein_max -, aniongap_min, aniongap_max -, bicarbonate_min, bicarbonate_max -, bun_min, bun_max -, calcium_min, calcium_max -, chloride_min, chloride_max -, creatinine_min, creatinine_max -, glucose_min, glucose_max -, sodium_min, sodium_max -, potassium_min, potassium_max --- blood differential -, basophils_abs_min, basophils_abs_max -, eosinophils_abs_min, eosinophils_abs_max -, lymphocytes_abs_min, lymphocytes_abs_max -, monocytes_abs_min, monocytes_abs_max -, neutrophils_abs_min, neutrophils_abs_max -, atypical_lymphocytes_min, atypical_lymphocytes_max -, bands_min, bands_max -, immature_granulocytes_min, immature_granulocytes_max -, metamyelocytes_min, metamyelocytes_max -, nrbc_min, nrbc_max --- coagulation -, d_dimer_min, d_dimer_max -, fibrinogen_min, fibrinogen_max -, thrombin_min, thrombin_max -, inr_min, inr_max -, pt_min, pt_max -, ptt_min, ptt_max --- enzymes and bilirubin -, alt_min, alt_max -, alp_min, alp_max -, ast_min, ast_max -, amylase_min, amylase_max -, bilirubin_total_min, bilirubin_total_max -, bilirubin_direct_min, bilirubin_direct_max -, bilirubin_indirect_min, bilirubin_indirect_max -, ck_cpk_min, ck_cpk_max -, ck_mb_min, ck_mb_max -, ggt_min, ggt_max -, ld_ldh_min, ld_ldh_max -FROM mimiciv_icu.icustays ie -LEFT JOIN cbc - ON ie.stay_id = cbc.stay_id -LEFT JOIN chem - ON ie.stay_id = chem.stay_id -LEFT JOIN diff - ON ie.stay_id = diff.stay_id -LEFT JOIN coag - ON ie.stay_id = coag.stay_id -LEFT JOIN enz - ON ie.stay_id = enz.stay_id -; diff --git a/mimic-iv/concepts_postgres/demographics/icustay_detail.sql b/mimic-iv/concepts_postgres/demographics/icustay_detail.sql index f20dbd77c..1a4c55c7b 100644 --- a/mimic-iv/concepts_postgres/demographics/icustay_detail.sql +++ b/mimic-iv/concepts_postgres/demographics/icustay_detail.sql @@ -21,7 +21,7 @@ SELECT ie.intime AS icu_intime, ie.outtime AS icu_outtime, ROUND( - CAST(CAST(CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', ie.outtime) - DATE_TRUNC('hour', ie.intime)) / 3600 AS BIGINT) AS DOUBLE PRECISION) / 24.0 AS DECIMAL), + CAST(CAST(CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', ie.outtime) - DATE_TRUNC('hour', ie.intime)) / 3600 AS BIGINT) AS DOUBLE PRECISION) / 24.0 AS DECIMAL(38, 9)), 2 ) AS los_icu, DENSE_RANK() OVER (PARTITION BY ie.hadm_id ORDER BY ie.intime NULLS FIRST) AS icustay_seq, /* first ICU stay *for the current hospitalization* */ diff --git a/mimic-iv/concepts_postgres/demographics/weight_durations.sql b/mimic-iv/concepts_postgres/demographics/weight_durations.sql index cca044feb..4edf740b7 100644 --- a/mimic-iv/concepts_postgres/demographics/weight_durations.sql +++ b/mimic-iv/concepts_postgres/demographics/weight_durations.sql @@ -5,13 +5,14 @@ WITH wt_stg AS ( SELECT c.stay_id, c.charttime, - CASE WHEN c.itemid = 226512 THEN 'admit' ELSE 'daily' END AS weight_type, /* TODO: eliminate obvious outliers if there is a reasonable weight */ - c.valuenum AS weight + CASE WHEN c.itemid = 226512 THEN 'admit' ELSE 'daily' END AS weight_type, + ROUND(CAST(c.valuenum AS DECIMAL(38, 9)), 3) AS weight FROM mimiciv_icu.chartevents AS c WHERE NOT c.valuenum IS NULL AND c.itemid IN (226512, /* Admit Wt */224639 /* Daily Weight */) AND c.valuenum > 0 + AND c.valuenum < 1500 ), wt_stg1 /* assign ascending row number */ AS ( SELECT stay_id, diff --git a/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql b/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql index 67155d822..4f42e0260 100644 --- a/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql +++ b/mimic-iv/concepts_postgres/firstday/first_day_gcs.sql @@ -9,8 +9,8 @@ WITH gcs_final AS ( g.gcs_motor, g.gcs_verbal, g.gcs_eyes, - g.gcs_unable, /* This sorts the data by GCS */ /* rn = 1 is the the lowest total GCS value */ - ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs NULLS FIRST) AS gcs_seq + g.gcs_unable, /* This sorts the data by GCS */ /* rn = 1 is the the lowest total GCS value */ /* tie-break on charttime (unique per stay) so the component columns */ /* are deterministic across SQL engines when multiple measurements */ /* share the same minimum GCS */ + ROW_NUMBER() OVER (PARTITION BY g.stay_id ORDER BY g.gcs ASC, g.charttime DESC NULLS LAST) AS gcs_seq FROM mimiciv_icu.icustays AS ie /* Only get data for the first 24 hours */ LEFT JOIN mimiciv_derived.gcs AS g diff --git a/mimic-iv/concepts_postgres/firstday/first_day_height.sql b/mimic-iv/concepts_postgres/firstday/first_day_height.sql index 14f765f94..92da9a24b 100644 --- a/mimic-iv/concepts_postgres/firstday/first_day_height.sql +++ b/mimic-iv/concepts_postgres/firstday/first_day_height.sql @@ -4,7 +4,7 @@ DROP TABLE IF EXISTS mimiciv_derived.first_day_height; CREATE TABLE mimiciv_deri SELECT ie.subject_id, ie.stay_id, - ROUND(CAST(AVG(height) AS DECIMAL), 2) AS height + ROUND(CAST(AVG(height) AS DECIMAL(38, 9)), 2) AS height FROM mimiciv_icu.icustays AS ie LEFT JOIN mimiciv_derived.height AS ht ON ie.stay_id = ht.stay_id diff --git a/mimic-iv/concepts_postgres/firstday/first_day_rrt.sql b/mimic-iv/concepts_postgres/firstday/first_day_rrt.sql index 1b7b231bd..904bcf2a4 100644 --- a/mimic-iv/concepts_postgres/firstday/first_day_rrt.sql +++ b/mimic-iv/concepts_postgres/firstday/first_day_rrt.sql @@ -6,7 +6,9 @@ SELECT ie.stay_id, MAX(dialysis_present) AS dialysis_present, MAX(dialysis_active) AS dialysis_active, - STRING_AGG(DISTINCT dialysis_type, ', ') AS dialysis_type + STRING_AGG(DISTINCT dialysis_type, ', ' + ORDER BY + dialysis_type NULLS FIRST) AS dialysis_type FROM mimiciv_icu.icustays AS ie LEFT JOIN mimiciv_derived.rrt AS rrt ON ie.stay_id = rrt.stay_id diff --git a/mimic-iv/concepts_postgres/measurement/bg.sql b/mimic-iv/concepts_postgres/measurement/bg.sql index 994c4bf16..dca59ee3a 100644 --- a/mimic-iv/concepts_postgres/measurement/bg.sql +++ b/mimic-iv/concepts_postgres/measurement/bg.sql @@ -115,7 +115,7 @@ WITH bg AS ( ), stg2 AS ( SELECT bg.*, - ROW_NUMBER() OVER (PARTITION BY bg.subject_id, bg.charttime ORDER BY s1.charttime DESC NULLS LAST) AS lastrowspo2, + ROW_NUMBER() OVER (PARTITION BY bg.specimen_id ORDER BY s1.charttime DESC NULLS LAST) AS lastrowspo2, s1.spo2 FROM bg LEFT JOIN stg_spo2 AS s1 @@ -126,7 +126,7 @@ WITH bg AS ( ), stg3 AS ( SELECT bg.*, - ROW_NUMBER() OVER (PARTITION BY bg.subject_id, bg.charttime ORDER BY s2.charttime DESC NULLS LAST) AS lastrowfio2, + ROW_NUMBER() OVER (PARTITION BY bg.specimen_id ORDER BY s2.charttime DESC NULLS LAST) AS lastrowfio2, s2.fio2_chartevents FROM stg2 AS bg LEFT JOIN stg_fio2 AS s2 @@ -148,28 +148,31 @@ SELECT pco2, fio2_chartevents, fio2, - aado2, /* also calculate AADO2 */ - CASE - WHEN po2 IS NULL OR pco2 IS NULL - THEN NULL - WHEN NOT fio2 IS NULL - THEN ( - CAST(fio2 AS DOUBLE PRECISION) / 100 - ) * ( - 760 - 47 - ) - ( - CAST(pco2 AS DOUBLE PRECISION) / 0.8 - ) - po2 - WHEN NOT fio2_chartevents IS NULL - THEN ( - CAST(fio2_chartevents AS DOUBLE PRECISION) / 100 - ) * ( - 760 - 47 - ) - ( - CAST(pco2 AS DOUBLE PRECISION) / 0.8 - ) - po2 - ELSE NULL - END AS aado2_calc, + aado2, /* also calculate AADO2, rounded to 4 decimal places */ + ROUND( + CAST(CASE + WHEN po2 IS NULL OR pco2 IS NULL + THEN NULL + WHEN NOT fio2 IS NULL + THEN ( + CAST(fio2 AS DOUBLE PRECISION) / 100 + ) * ( + 760 - 47 + ) - ( + CAST(pco2 AS DOUBLE PRECISION) / 0.8 + ) - po2 + WHEN NOT fio2_chartevents IS NULL + THEN ( + CAST(fio2_chartevents AS DOUBLE PRECISION) / 100 + ) * ( + 760 - 47 + ) - ( + CAST(pco2 AS DOUBLE PRECISION) / 0.8 + ) - po2 + ELSE NULL + END AS DECIMAL(38, 9)), + 4 + ) AS aado2_calc, CASE WHEN po2 IS NULL THEN NULL diff --git a/mimic-iv/concepts_postgres/measurement/blood_differential.sql b/mimic-iv/concepts_postgres/measurement/blood_differential.sql index 8e3395792..73f906e1d 100644 --- a/mimic-iv/concepts_postgres/measurement/blood_differential.sql +++ b/mimic-iv/concepts_postgres/measurement/blood_differential.sql @@ -109,7 +109,7 @@ SELECT WHEN basophils_abs IS NULL AND NOT basophils IS NULL AND impute_abs = 1 THEN CAST(basophils * wbc AS DOUBLE PRECISION) / 100 ELSE basophils_abs - END AS DECIMAL), + END AS DECIMAL(38, 9)), 4 ) AS basophils_abs, ROUND( @@ -117,7 +117,7 @@ SELECT WHEN eosinophils_abs IS NULL AND NOT eosinophils IS NULL AND impute_abs = 1 THEN CAST(eosinophils * wbc AS DOUBLE PRECISION) / 100 ELSE eosinophils_abs - END AS DECIMAL), + END AS DECIMAL(38, 9)), 4 ) AS eosinophils_abs, ROUND( @@ -125,7 +125,7 @@ SELECT WHEN lymphocytes_abs IS NULL AND NOT lymphocytes IS NULL AND impute_abs = 1 THEN CAST(lymphocytes * wbc AS DOUBLE PRECISION) / 100 ELSE lymphocytes_abs - END AS DECIMAL), + END AS DECIMAL(38, 9)), 4 ) AS lymphocytes_abs, ROUND( @@ -133,7 +133,7 @@ SELECT WHEN monocytes_abs IS NULL AND NOT monocytes IS NULL AND impute_abs = 1 THEN CAST(monocytes * wbc AS DOUBLE PRECISION) / 100 ELSE monocytes_abs - END AS DECIMAL), + END AS DECIMAL(38, 9)), 4 ) AS monocytes_abs, ROUND( @@ -141,7 +141,7 @@ SELECT WHEN neutrophils_abs IS NULL AND NOT neutrophils IS NULL AND impute_abs = 1 THEN CAST(neutrophils * wbc AS DOUBLE PRECISION) / 100 ELSE neutrophils_abs - END AS DECIMAL), + END AS DECIMAL(38, 9)), 4 ) AS neutrophils_abs, basophils, diff --git a/mimic-iv/concepts_postgres/measurement/height.sql b/mimic-iv/concepts_postgres/measurement/height.sql index e5f33b76b..512424170 100644 --- a/mimic-iv/concepts_postgres/measurement/height.sql +++ b/mimic-iv/concepts_postgres/measurement/height.sql @@ -6,7 +6,7 @@ WITH ht_in AS ( c.subject_id, c.stay_id, c.charttime, /* Ensure that all heights are in centimeters */ - ROUND(CAST(c.valuenum * 2.54 AS DECIMAL), 2) AS height, + ROUND(CAST(c.valuenum * 2.54 AS DECIMAL(38, 9)), 2) AS height, c.valuenum AS height_orig FROM mimiciv_icu.chartevents AS c WHERE @@ -16,7 +16,7 @@ WITH ht_in AS ( c.subject_id, c.stay_id, c.charttime, /* Ensure that all heights are in centimeters */ - ROUND(CAST(c.valuenum AS DECIMAL), 2) AS height + ROUND(CAST(c.valuenum AS DECIMAL(38, 9)), 2) AS height FROM mimiciv_icu.chartevents AS c WHERE NOT c.valuenum IS NULL AND /* Height cm */ c.itemid = 226730 diff --git a/mimic-iv/concepts_postgres/measurement/kdigo_uo.sql b/mimic-iv/concepts_postgres/measurement/kdigo_uo.sql deleted file mode 100644 index 645a970d7..000000000 --- a/mimic-iv/concepts_postgres/measurement/kdigo_uo.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE IF EXISTS kdigo_uo; CREATE TABLE kdigo_uo AS -DROP TABLE IF EXISTS kdigo_uo; CREATE TABLE kdigo_uo AS diff --git a/mimic-iv/concepts_postgres/measurement/oxygen_delivery.sql b/mimic-iv/concepts_postgres/measurement/oxygen_delivery.sql index b23f7c530..d35942373 100644 --- a/mimic-iv/concepts_postgres/measurement/oxygen_delivery.sql +++ b/mimic-iv/concepts_postgres/measurement/oxygen_delivery.sql @@ -26,8 +26,11 @@ WITH ce_stg1 AS ( itemid, value, valuenum, - valueuom, /* retain only 1 row per charttime */ /* prioritizing the last documented value */ /* primarily used to subselect o2 flows */ - ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY storetime DESC NULLS LAST) AS rn + valueuom, /* retain only 1 row per charttime */ /* prioritizing (1) the last documented value */ /* and (2) the highest value */ /* primarily used to subselect o2 flows */ /* when storetime ties (e.g. an o2 flow 223834 and a bipap o2 flow */ /* 227582 documented together), prefer the highest value so the result */ /* is deterministic across SQL engines */ + ROW_NUMBER() OVER ( + PARTITION BY subject_id, charttime, itemid + ORDER BY storetime DESC NULLS LAST, valuenum DESC NULLS LAST + ) AS rn FROM ce_stg1 AS ce ), o2 AS ( /* The below ITEMID can have multiple entries for charttime/storetime */ /* These are valid entries, and should be retained in derived tables. */ /* 224181 -- Small Volume Neb Drug #1 | Respiratory | Text */ /* , 227570 -- Small Volume Neb Drug/Dose #1 | Respiratory | Text */ /* , 224833 -- SBT Deferred | Respiratory | Text */ /* , 224716 -- SBT Stopped | Respiratory | Text */ /* , 224740 -- RSBI Deferred | Respiratory | Text */ /* , 224829 -- Trach Tube Type | Respiratory | Text */ /* , 226732 -- O2 Delivery Device(s) | Respiratory | Text */ /* , 226873 -- Inspiratory Ratio | Respiratory | Numeric */ /* , 226871 -- Expiratory Ratio | Respiratory | Numeric */ /* maximum of 4 o2 devices on at once */ @@ -37,7 +40,10 @@ WITH ce_stg1 AS ( charttime, itemid, value AS o2_device, - ROW_NUMBER() OVER (PARTITION BY subject_id, charttime, itemid ORDER BY value NULLS FIRST) AS rn + ROW_NUMBER() OVER ( + PARTITION BY subject_id, charttime, itemid + ORDER BY storetime DESC NULLS LAST, value DESC NULLS LAST + ) AS rn FROM mimiciv_icu.chartevents WHERE itemid = 226732 /* oxygen delivery device(s) */ diff --git a/mimic-iv/concepts_postgres/measurement/rhythm.sql b/mimic-iv/concepts_postgres/measurement/rhythm.sql index 243410f63..7eba65d0d 100644 --- a/mimic-iv/concepts_postgres/measurement/rhythm.sql +++ b/mimic-iv/concepts_postgres/measurement/rhythm.sql @@ -4,7 +4,11 @@ DROP TABLE IF EXISTS mimiciv_derived.rhythm; CREATE TABLE mimiciv_derived.rhythm SELECT ce.subject_id, ce.charttime, - MAX(CASE WHEN itemid = 220048 THEN value ELSE NULL END) AS heart_rhythm, + STRING_AGG( + DISTINCT CASE WHEN itemid = 220048 THEN value ELSE NULL END, '; ' + ORDER BY + CASE WHEN itemid = 220048 THEN value ELSE NULL END NULLS FIRST + ) AS heart_rhythm, MAX(CASE WHEN itemid = 224650 THEN value ELSE NULL END) AS ectopy_type, MAX(CASE WHEN itemid = 224651 THEN value ELSE NULL END) AS ectopy_frequency, MAX(CASE WHEN itemid = 226479 THEN value ELSE NULL END) AS ectopy_type_secondary, diff --git a/mimic-iv/concepts_postgres/measurement/urine_output_rate.sql b/mimic-iv/concepts_postgres/measurement/urine_output_rate.sql index a58565789..4cdc9ef7e 100644 --- a/mimic-iv/concepts_postgres/measurement/urine_output_rate.sql +++ b/mimic-iv/concepts_postgres/measurement/urine_output_rate.sql @@ -40,13 +40,16 @@ WITH tm AS ( ELSE NULL END ) AS urineoutput_6hr, - CAST(SUM( - CASE - WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', io.charttime) - DATE_TRUNC('hour', iosum.charttime)) / 3600 AS BIGINT) <= 5 - THEN iosum.tm_since_last_uo - ELSE NULL - END - ) AS DOUBLE PRECISION) / 60.0 AS uo_tm_6hr, + ROUND( + CAST(CAST(SUM( + CASE + WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', io.charttime) - DATE_TRUNC('hour', iosum.charttime)) / 3600 AS BIGINT) <= 5 + THEN iosum.tm_since_last_uo + ELSE NULL + END + ) AS DOUBLE PRECISION) / 60.0 AS DECIMAL(38, 9)), + 6 + ) AS uo_tm_6hr, SUM( CASE WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', io.charttime) - DATE_TRUNC('hour', iosum.charttime)) / 3600 AS BIGINT) <= 11 @@ -54,15 +57,21 @@ WITH tm AS ( ELSE NULL END ) AS urineoutput_12hr, - CAST(SUM( - CASE - WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', io.charttime) - DATE_TRUNC('hour', iosum.charttime)) / 3600 AS BIGINT) <= 11 - THEN iosum.tm_since_last_uo - ELSE NULL - END - ) AS DOUBLE PRECISION) / 60.0 AS uo_tm_12hr, /* 24 hours */ + ROUND( + CAST(CAST(SUM( + CASE + WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', io.charttime) - DATE_TRUNC('hour', iosum.charttime)) / 3600 AS BIGINT) <= 11 + THEN iosum.tm_since_last_uo + ELSE NULL + END + ) AS DOUBLE PRECISION) / 60.0 AS DECIMAL(38, 9)), + 6 + ) AS uo_tm_12hr, /* 24 hours */ SUM(iosum.urineoutput) AS urineoutput_24hr, - CAST(SUM(iosum.tm_since_last_uo) AS DOUBLE PRECISION) / 60.0 AS uo_tm_24hr + ROUND( + CAST(CAST(SUM(iosum.tm_since_last_uo) AS DOUBLE PRECISION) / 60.0 AS DECIMAL(38, 9)), + 6 + ) AS uo_tm_24hr FROM uo_tm AS io /* this join gives you all UO measurements over a 24 hour period */ LEFT JOIN uo_tm AS iosum @@ -88,7 +97,7 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_6hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_6hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) END AS uo_mlkghr_6hr, @@ -97,7 +106,7 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_12hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_12hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) END AS uo_mlkghr_12hr, @@ -106,13 +115,13 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_24hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_24hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) END AS uo_mlkghr_24hr, /* time of earliest UO measurement that was used to calculate the rate */ - ROUND(CAST(uo_tm_6hr AS DECIMAL), 2) AS uo_tm_6hr, - ROUND(CAST(uo_tm_12hr AS DECIMAL), 2) AS uo_tm_12hr, - ROUND(CAST(uo_tm_24hr AS DECIMAL), 2) AS uo_tm_24hr + ROUND(CAST(uo_tm_6hr AS DECIMAL(38, 9)), 2) AS uo_tm_6hr, + ROUND(CAST(uo_tm_12hr AS DECIMAL(38, 9)), 2) AS uo_tm_12hr, + ROUND(CAST(uo_tm_24hr AS DECIMAL(38, 9)), 2) AS uo_tm_24hr FROM ur_stg AS ur LEFT JOIN mimiciv_derived.weight_durations AS wd ON ur.stay_id = wd.stay_id diff --git a/mimic-iv/concepts_postgres/measurement/vitalsign.sql b/mimic-iv/concepts_postgres/measurement/vitalsign.sql index 47c397cd0..4646ccb0c 100644 --- a/mimic-iv/concepts_postgres/measurement/vitalsign.sql +++ b/mimic-iv/concepts_postgres/measurement/vitalsign.sql @@ -45,7 +45,7 @@ SELECT WHEN itemid IN (223762) AND valuenum > 10 AND valuenum < 50 THEN valuenum END - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 2 ) AS temperature, MAX(CASE WHEN itemid = 224642 THEN value END) AS temperature_site, diff --git a/mimic-iv/concepts_postgres/medication/norepinephrine_equivalent_dose.sql b/mimic-iv/concepts_postgres/medication/norepinephrine_equivalent_dose.sql index b63a0ebec..883e6d881 100644 --- a/mimic-iv/concepts_postgres/medication/norepinephrine_equivalent_dose.sql +++ b/mimic-iv/concepts_postgres/medication/norepinephrine_equivalent_dose.sql @@ -6,7 +6,7 @@ SELECT starttime, endtime, /* calculate the dose */ /* all sources are in mcg/kg/min, */ /* except vasopressin which is in units/hour */ ROUND( - CAST(COALESCE(norepinephrine, 0) + COALESCE(epinephrine, 0) + COALESCE(CAST(phenylephrine AS DOUBLE PRECISION) / 10, 0) + COALESCE(CAST(dopamine AS DOUBLE PRECISION) / 100, 0) + /* + metaraminol/8 -- metaraminol not used in BIDMC */ COALESCE(CAST(vasopressin * 2.5 AS DOUBLE PRECISION) / 60, 0) AS DECIMAL), + CAST(COALESCE(norepinephrine, 0) + COALESCE(epinephrine, 0) + COALESCE(CAST(phenylephrine AS DOUBLE PRECISION) / 10, 0) + COALESCE(CAST(dopamine AS DOUBLE PRECISION) / 100, 0) + /* + metaraminol/8 -- metaraminol not used in BIDMC */ COALESCE(CAST(vasopressin * 2.5 AS DOUBLE PRECISION) / 60, 0) AS DECIMAL(38, 9)), 4 ) AS norepinephrine_equivalent_dose FROM mimiciv_derived.vasoactive_agent diff --git a/mimic-iv/concepts_postgres/organfailure/kdigo_uo.sql b/mimic-iv/concepts_postgres/organfailure/kdigo_uo.sql index 22466fa27..e514dcc0b 100644 --- a/mimic-iv/concepts_postgres/organfailure/kdigo_uo.sql +++ b/mimic-iv/concepts_postgres/organfailure/kdigo_uo.sql @@ -34,20 +34,29 @@ WITH uo_stg1 AS ( ORDER BY seconds_since_admit NULLS FIRST RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW ) AS urineoutput_24hr, /* repeat the summations using the hours_since_previous_row column */ /* this gives us the amount of time the UO was calculated over */ - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 21600 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 21600 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_6hr, - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 43200 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 43200 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_12hr, - SUM(hours_since_previous_row) OVER ( - PARTITION BY stay_id - ORDER BY seconds_since_admit NULLS FIRST - RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW + ROUND( + CAST(SUM(hours_since_previous_row) OVER ( + PARTITION BY stay_id + ORDER BY seconds_since_admit NULLS FIRST + RANGE BETWEEN 86400 PRECEDING AND CURRENT ROW + ) AS DECIMAL(38, 9)), + 6 ) AS uo_tm_24hr FROM uo_stg1 ) @@ -63,7 +72,7 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_6hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_6hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) ELSE NULL @@ -73,7 +82,7 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_12hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_12hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) ELSE NULL @@ -83,7 +92,7 @@ SELECT THEN ROUND( CAST(( CAST(CAST(ur.urineoutput_24hr AS DOUBLE PRECISION) / wd.weight AS DOUBLE PRECISION) / uo_tm_24hr - ) AS DECIMAL), + ) AS DECIMAL(38, 9)), 4 ) ELSE NULL diff --git a/mimic-iv/concepts_postgres/organfailure/meld.sql b/mimic-iv/concepts_postgres/organfailure/meld.sql index 9d2f51303..f6c28ebba 100644 --- a/mimic-iv/concepts_postgres/organfailure/meld.sql +++ b/mimic-iv/concepts_postgres/organfailure/meld.sql @@ -83,7 +83,7 @@ WITH cohort AS ( creatinine_score + bilirubin_score + inr_score ) > 4 THEN 40.0 - ELSE ROUND(CAST(creatinine_score + bilirubin_score + inr_score AS DECIMAL), 1) * 10 + ELSE ROUND(CAST(creatinine_score + bilirubin_score + inr_score AS DECIMAL(38, 9)), 1) * 10 END AS meld_initial FROM score ) diff --git a/mimic-iv/concepts_postgres/postgres-concept-index.sql b/mimic-iv/concepts_postgres/postgres-concept-index.sql new file mode 100644 index 000000000..8decebc72 --- /dev/null +++ b/mimic-iv/concepts_postgres/postgres-concept-index.sql @@ -0,0 +1,74 @@ +---------------------------------------------------- +---------------------------------------------------- +-- Indexes for the mimiciv_derived concept tables -- +---------------------------------------------------- +---------------------------------------------------- + +-- The score/firstday concepts join hourly windows against the measurement +-- and medication concepts using (stay_id/hadm_id/subject_id, charttime) +-- range predicates. Without these indexes PostgreSQL falls back to +-- merge/hash joins that materialise hundreds of millions of intermediate +-- rows. With them the joins become per-hour index lookups and the same +-- scripts run in minutes. + +SET search_path TO mimiciv_derived; + +DROP INDEX IF EXISTS icustay_hourly_idx01; +CREATE INDEX icustay_hourly_idx01 + ON icustay_hourly (stay_id); + +-- measurement concepts joined on stay_id + charttime + +DROP INDEX IF EXISTS vitalsign_idx01; +CREATE INDEX vitalsign_idx01 + ON vitalsign (stay_id, charttime); + +DROP INDEX IF EXISTS gcs_idx01; +CREATE INDEX gcs_idx01 + ON gcs (stay_id, charttime); + +DROP INDEX IF EXISTS urine_output_rate_idx01; +CREATE INDEX urine_output_rate_idx01 + ON urine_output_rate (stay_id, charttime); + +-- measurement concepts joined on hadm_id + charttime + +DROP INDEX IF EXISTS chemistry_idx01; +CREATE INDEX chemistry_idx01 + ON chemistry (hadm_id, charttime); + +DROP INDEX IF EXISTS complete_blood_count_idx01; +CREATE INDEX complete_blood_count_idx01 + ON complete_blood_count (hadm_id, charttime); + +DROP INDEX IF EXISTS enzyme_idx01; +CREATE INDEX enzyme_idx01 + ON enzyme (hadm_id, charttime); + +-- blood gas joined on subject_id + charttime + +DROP INDEX IF EXISTS bg_idx01; +CREATE INDEX bg_idx01 + ON bg (subject_id, charttime); + +-- ventilation and vasopressor concepts joined on stay_id + starttime/endtime + +DROP INDEX IF EXISTS ventilation_idx01; +CREATE INDEX ventilation_idx01 + ON ventilation (stay_id, starttime, endtime); + +DROP INDEX IF EXISTS epinephrine_idx01; +CREATE INDEX epinephrine_idx01 + ON epinephrine (stay_id, starttime, endtime); + +DROP INDEX IF EXISTS norepinephrine_idx01; +CREATE INDEX norepinephrine_idx01 + ON norepinephrine (stay_id, starttime, endtime); + +DROP INDEX IF EXISTS dopamine_idx01; +CREATE INDEX dopamine_idx01 + ON dopamine (stay_id, starttime, endtime); + +DROP INDEX IF EXISTS dobutamine_idx01; +CREATE INDEX dobutamine_idx01 + ON dobutamine (stay_id, starttime, endtime); diff --git a/mimic-iv/concepts_postgres/postgres-make-concepts.sql b/mimic-iv/concepts_postgres/postgres-make-concepts.sql index 3bbed1e3f..7d2d9bda5 100644 --- a/mimic-iv/concepts_postgres/postgres-make-concepts.sql +++ b/mimic-iv/concepts_postgres/postgres-make-concepts.sql @@ -1,8 +1,8 @@ \echo '' \echo '===' -\echo 'Beginning to create materialized views for MIMIC database.' -\echo 'Any notices of the form "NOTICE: materialized view "XXXXXX" does not exist" can be ignored.' -\echo 'The scripts drop views before creating them, and these notices indicate nothing existed prior to creating the view.' +\echo 'Beginning to create derived concepts for MIMIC-IV.' +\echo 'Any notices of the form "NOTICE: table "XXXXXX" does not exist" can be ignored.' +\echo 'The scripts drop tables before creating them, and these notices indicate nothing existed.' \echo '===' \echo '' @@ -49,6 +49,7 @@ SET search_path TO mimiciv_derived, mimiciv_hosp, mimiciv_icu, mimiciv_ed; -- medication \i medication/acei.sql \i medication/antibiotic.sql +\i medication/arb.sql \i medication/dobutamine.sql \i medication/dopamine.sql \i medication/epinephrine.sql @@ -60,6 +61,7 @@ SET search_path TO mimiciv_derived, mimiciv_hosp, mimiciv_icu, mimiciv_ed; \i medication/vasopressin.sql -- treatment +\i treatment/code_status.sql \i treatment/crrt.sql \i treatment/invasive_line.sql \i treatment/rrt.sql @@ -80,6 +82,10 @@ SET search_path TO mimiciv_derived, mimiciv_hosp, mimiciv_icu, mimiciv_ed; \i organfailure/kdigo_creatinine.sql \i organfailure/meld.sql +-- indexes on the concept tables above, required for the hourly score +-- queries (sofa, apsiii, ...) to run in minutes rather than hours +\i postgres-concept-index.sql + -- score \i score/apsiii.sql \i score/lods.sql diff --git a/mimic-iv/concepts_postgres/sepsis/suspicion_of_infection.sql b/mimic-iv/concepts_postgres/sepsis/suspicion_of_infection.sql index 40c548fd6..67ff3c070 100644 --- a/mimic-iv/concepts_postgres/sepsis/suspicion_of_infection.sql +++ b/mimic-iv/concepts_postgres/sepsis/suspicion_of_infection.sql @@ -9,10 +9,10 @@ WITH ab_tbl AS ( abx.antibiotic, abx.starttime AS antibiotic_time, /* date is used to match microbiology cultures with only date available */ DATE_TRUNC('day', abx.starttime) AS antibiotic_date, - abx.stoptime AS antibiotic_stoptime, /* create a unique identifier for each patient antibiotic */ + abx.stoptime AS antibiotic_stoptime, /* create a unique identifier for each patient antibiotic */ /* hadm_id/stay_id tiebreak keeps the numbering deterministic when */ /* multiple antibiotics share the same start/stop time and name */ ROW_NUMBER() OVER ( PARTITION BY subject_id - ORDER BY starttime NULLS FIRST, stoptime NULLS FIRST, antibiotic NULLS FIRST + ORDER BY starttime NULLS FIRST, stoptime NULLS FIRST, antibiotic NULLS FIRST, hadm_id NULLS FIRST, stay_id NULLS FIRST ) AS ab_id FROM mimiciv_derived.antibiotic AS abx ), me AS ( @@ -42,10 +42,10 @@ WITH ab_tbl AS ( me72.micro_specimen_id, COALESCE(me72.charttime, CAST(me72.chartdate AS TIMESTAMP)) AS last72_charttime, me72.positiveculture AS last72_positiveculture, - me72.spec_type_desc AS last72_specimen, /* we will use this partition to select the earliest culture */ /* before this abx */ /* this ensures each antibiotic is only matched to a single culture */ /* and consequently we have 1 row per antibiotic */ + me72.spec_type_desc AS last72_specimen, /* we will use this partition to select the earliest culture */ /* before this abx */ /* this ensures each antibiotic is only matched to a single culture */ /* and consequently we have 1 row per antibiotic */ /* among cultures at the same time point, prefer a positive culture */ /* (positiveculture DESC) so that any positive culture yields */ /* positive_culture = 1, per the suspicion of infection criteria; the */ /* micro_specimen_id tiebreak makes the selection deterministic */ ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me72.chartdate NULLS FIRST, me72.charttime + ORDER BY me72.chartdate NULLS FIRST, me72.charttime, me72.positiveculture DESC NULLS LAST, me72.micro_specimen_id NULLS FIRST ) AS micro_seq FROM ab_tbl /* abx taken after culture, but no more than 72 hours after */ @@ -72,10 +72,10 @@ WITH ab_tbl AS ( me24.micro_specimen_id, COALESCE(me24.charttime, CAST(me24.chartdate AS TIMESTAMP)) AS next24_charttime, me24.positiveculture AS next24_positiveculture, - me24.spec_type_desc AS next24_specimen, /* we will use this partition to select the earliest culture */ /* before this abx */ /* this ensures each antibiotic is only matched to a single culture */ /* and consequently we have 1 row per antibiotic */ + me24.spec_type_desc AS next24_specimen, /* we will use this partition to select the earliest culture */ /* before this abx */ /* this ensures each antibiotic is only matched to a single culture */ /* and consequently we have 1 row per antibiotic */ /* among cultures at the same time point, prefer a positive culture */ /* (positiveculture DESC) so that any positive culture yields */ /* positive_culture = 1, per the suspicion of infection criteria; the */ /* micro_specimen_id tiebreak makes the selection deterministic */ ROW_NUMBER() OVER ( PARTITION BY ab_tbl.subject_id, ab_tbl.ab_id - ORDER BY me24.chartdate NULLS FIRST, me24.charttime + ORDER BY me24.chartdate NULLS FIRST, me24.charttime, me24.positiveculture DESC NULLS LAST, me24.micro_specimen_id NULLS FIRST ) AS micro_seq FROM ab_tbl /* culture in subsequent 24 hours */ diff --git a/src/mimic_utils/compare_concepts.py b/src/mimic_utils/compare_concepts.py index c0e03412d..2c345e9d6 100644 --- a/src/mimic_utils/compare_concepts.py +++ b/src/mimic_utils/compare_concepts.py @@ -55,12 +55,25 @@ def _list_tables_duckdb(con, schema): def _as_numeric(series): """Return a float Series if every non-null value coerces, else None.""" + if _is_datetime_like_series(series): + return None coerced = pd.to_numeric(series, errors="coerce") if coerced.isna().sum() == series.isna().sum(): return coerced.astype(float) return None +def _is_datetime_like_series(series): + """Return True when values are already datetimes, even if stored as object dtype.""" + if pd.api.types.is_datetime64_any_dtype(series.dtype) or pd.api.types.is_datetime64tz_dtype(series.dtype): + return True + non_null = series.dropna() + if non_null.empty: + return False + sample = non_null.iloc[0] + return isinstance(sample, (pd.Timestamp, np.datetime64)) or hasattr(sample, "tzinfo") + + def _as_datetime(series): """Return a datetime Series if every non-null value parses, else None.""" import warnings @@ -68,6 +81,8 @@ def _as_datetime(series): warnings.simplefilter("ignore") # silence per-element parse fallback notices coerced = pd.to_datetime(series, errors="coerce") if coerced.isna().sum() == series.isna().sum(): + if pd.api.types.is_datetime64tz_dtype(coerced.dtype): + coerced = coerced.dt.tz_localize(None) return coerced return None diff --git a/src/mimic_utils/sqlglot_dialects/duckdb.py b/src/mimic_utils/sqlglot_dialects/duckdb.py index 48e5e135c..9d282ad23 100644 --- a/src/mimic_utils/sqlglot_dialects/duckdb.py +++ b/src/mimic_utils/sqlglot_dialects/duckdb.py @@ -11,6 +11,8 @@ class MimicDuckDB(DuckDB): class Generator(DuckDB.Generator): def datatype_sql(self, expression: exp.DataType) -> str: + if expression.this == exp.DataType.Type.TIMESTAMPTZ: + return "TIMESTAMP" if expression.this == exp.DataType.Type.DECIMAL and not expression.expressions: return "DECIMAL(38, 9)" return super().datatype_sql(expression) diff --git a/src/mimic_utils/sqlglot_dialects/postgres.py b/src/mimic_utils/sqlglot_dialects/postgres.py index 1048fe8c1..02a2f93f0 100644 --- a/src/mimic_utils/sqlglot_dialects/postgres.py +++ b/src/mimic_utils/sqlglot_dialects/postgres.py @@ -80,6 +80,13 @@ def _generate_array_sql(self: Postgres.Generator, expression: exp.Expression) -> class MimicPostgres(Postgres): class Generator(Postgres.Generator): + def datatype_sql(self, expression: exp.DataType) -> str: + if expression.this == exp.DataType.Type.TIMESTAMPTZ: + return "TIMESTAMP" + if expression.this == exp.DataType.Type.DECIMAL and not expression.expressions: + return "DECIMAL(38, 9)" + return super().datatype_sql(expression) + TRANSFORMS = { **Postgres.Generator.TRANSFORMS, exp.DatetimeDiff: _datetime_diff_sql, diff --git a/src/mimic_utils/transpile.py b/src/mimic_utils/transpile.py index 8487456b8..514039402 100644 --- a/src/mimic_utils/transpile.py +++ b/src/mimic_utils/transpile.py @@ -1,5 +1,6 @@ import logging import os +import re from pathlib import Path from typing import Union @@ -25,6 +26,12 @@ _CATALOG_TO_REMOVE = "physionet-data" +def _strip_timezone_types(sql: str) -> str: + """Normalize any timezone-aware timestamp types back to naive TIMESTAMP.""" + sql = re.sub(r"\bTIMESTAMP\s+WITH\s+TIME\s+ZONE\b", "TIMESTAMP", sql) + return re.sub(r"\bTIMESTAMPTZ\b", "TIMESTAMP", sql) + + def _strip_catalog(parsed: exp.Expression) -> None: """Remove the ``physionet-data`` catalog from every table reference in place.""" for table in parsed.find_all(exp.Table): @@ -50,11 +57,12 @@ def transpile_query(query: str, source_dialect: str = "bigquery", destination_di # drop comments when targeting it. keep_comments = destination_dialect != "duckdb" - return parsed.sql( + sql = parsed.sql( dialect=_DESTINATION_DIALECTS[destination_dialect], pretty=True, comments=keep_comments, ) + return _strip_timezone_types(sql) def transpile_file( diff --git a/tests/test_compare_concepts.py b/tests/test_compare_concepts.py new file mode 100644 index 000000000..5ad98e28b --- /dev/null +++ b/tests/test_compare_concepts.py @@ -0,0 +1,22 @@ +from datetime import datetime, timedelta, timezone + +import pandas as pd + +from mimic_utils.compare_concepts import compare_table + + +def test_compare_table_treats_same_wall_time_as_equal_across_timezones(): + pg = pd.DataFrame({"charttime": [datetime(2150, 1, 1, 3, 30, tzinfo=timezone(timedelta(hours=2)))]}) + duck = pd.DataFrame({"charttime": [datetime(2150, 1, 1, 3, 30)]}) + + assert compare_table(pg, duck, 1e-6, 1e-9) == (True, "1 rows OK") + + +def test_compare_table_flags_different_wall_times_even_if_instants_match(): + pg = pd.DataFrame({"charttime": [datetime(2150, 1, 1, 3, 30, tzinfo=timezone(timedelta(hours=2)))]}) + duck = pd.DataFrame({"charttime": [datetime(2150, 1, 1, 1, 30)]}) + + assert compare_table(pg, duck, 1e-6, 1e-9) == ( + False, + "1 differing cells in columns: charttime(1)", + ) \ No newline at end of file diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 67470da5f..20adcbb5f 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -72,12 +72,10 @@ def t(bq: str, dialect: str) -> str: ("int64_cast_pg", "SELECT CAST(hr AS INT64) AS hr FROM t", "postgres", "SELECT CAST(hr AS BIGINT) AS hr FROM t"), - # BigQuery NUMERIC == DECIMAL(38, 9); postgres NUMERIC is arbitrary precision - # (keeps the value), but DuckDB's bare DECIMAL defaults to scale 3 and rounds, - # so we must pin the precision there. - # postgres DECIMAL == NUMERIC, both arbitrary precision, so the value is kept + # BigQuery NUMERIC == DECIMAL(38, 9), so both target dialects must pin the + # precision and scale explicitly to preserve BigQuery semantics. ("numeric_cast_pg", "SELECT CAST(x AS NUMERIC) FROM t", "postgres", - "SELECT CAST(x AS DECIMAL) FROM t"), + "SELECT CAST(x AS DECIMAL(38, 9)) FROM t"), ("numeric_cast_duckdb", "SELECT CAST(x AS NUMERIC) FROM t", "duckdb", "SELECT CAST(x AS DECIMAL(38, 9)) FROM t"), ] @@ -102,6 +100,18 @@ def test_duckdb_comments_stripped(): assert "/*" in transpile_query(bq, "bigquery", "postgres") +@pytest.mark.parametrize( + "bq", + [ + "SELECT TIMESTAMP('2150-01-01 00:00:00') AS ts", + "SELECT CAST(DATETIME '2150-01-01 00:00:00' AS TIMESTAMP) AS ts", + ], +) +def test_timestamp_outputs_stay_timezone_naive(bq): + for dialect in ("postgres", "duckdb"): + assert "TIMESTAMPTZ" not in transpile_query(bq, "bigquery", dialect) + + def test_unsupported_dialect_raises(): with pytest.raises(ValueError): transpile_query("SELECT 1", "bigquery", "mysql")