From c06aeec351bafa49e88006a03b77ba3a1f04642e Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:22:58 +0300 Subject: [PATCH 01/49] fix: SECOND-based DATETIME_DIFF in urine_output_rate Match kdigo_uo so BigQuery truncation and Postgres fractional diffs no longer diverge on the 6/12h rate windows (#1549). Co-authored-by: Cursor --- .../measurement/urine_output_rate.sql | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/mimic-iv/concepts/measurement/urine_output_rate.sql b/mimic-iv/concepts/measurement/urine_output_rate.sql index 01939211..bc674014 100644 --- a/mimic-iv/concepts/measurement/urine_output_rate.sql +++ b/mimic-iv/concepts/measurement/urine_output_rate.sql @@ -16,12 +16,14 @@ WITH tm AS ( ) -- now calculate time since last UO measurement +-- Use SECOND diffs (like kdigo_uo) so BigQuery and PostgreSQL agree (#1549). +-- BigQuery DATETIME_DIFF(..., MINUTE/HOUR) truncates; SECOND keeps fractions. , uo_tm AS ( SELECT tm.stay_id , CASE WHEN LAG(charttime) OVER w IS NULL - THEN DATETIME_DIFF(charttime, intime_hr, MINUTE) - ELSE DATETIME_DIFF(charttime, LAG(charttime) OVER w, MINUTE) + THEN DATETIME_DIFF(charttime, intime_hr, SECOND) / 60.0 + ELSE DATETIME_DIFF(charttime, LAG(charttime) OVER w, SECOND) / 60.0 END AS tm_since_last_uo , uo.charttime , uo.urineoutput @@ -45,24 +47,33 @@ WITH tm AS ( -- to 1 hour of UO, therefore we use '5' and '11' to restrict the -- period, rather than 6/12 this assumption may overestimate UO rate -- when documentation is done less than hourly - , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 + , SUM(CASE + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 5 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_6hr , ROUND(CAST( SUM( CASE - WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 5 THEN iosum.tm_since_last_uo ELSE NULL END) / 60.0 AS NUMERIC ), 6) AS uo_tm_6hr - , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 + , SUM(CASE + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 11 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_12hr , ROUND(CAST( SUM( CASE - WHEN - DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 11 THEN iosum.tm_since_last_uo ELSE NULL END) / 60.0 AS NUMERIC ), 6) AS uo_tm_12hr From 5c49bc45ce9670917d55637ad5ffbddbe29df991 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:23:45 +0300 Subject: [PATCH 02/49] fix: escape password and quote ids in create_mimic_user Passwords with quotes and unusual user/db names broke the CREATE USER / DATABASE statements. Co-authored-by: Cursor --- .../buildmimic/postgres/create_mimic_user.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mimic-iii/buildmimic/postgres/create_mimic_user.sh b/mimic-iii/buildmimic/postgres/create_mimic_user.sh index 772764ce..d639f5fc 100755 --- a/mimic-iii/buildmimic/postgres/create_mimic_user.sh +++ b/mimic-iii/buildmimic/postgres/create_mimic_user.sh @@ -20,6 +20,13 @@ else echo "User is set to '$MIMIC_USER'"; fi +# escape ' in password for SQL string literal +MIMIC_PASSWORD_SQL=$(printf '%s' "$MIMIC_PASSWORD" | sed "s/'/''/g") +# quote SQL identifiers (double any embedded ") +sql_ident () { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; } +MIMIC_USER_SQL=$(sql_ident "$MIMIC_USER") +MIMIC_DB_SQL=$(sql_ident "$MIMIC_DB") + PSQL='psql' # add in the host/port, if they were specified (not null, -n) @@ -58,11 +65,11 @@ fi if [ "$MIMIC_USER" != "postgres" ]; then # we need to create this user via postgres # use SUDO to login as postgres - $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER; CREATE USER $MIMIC_USER WITH PASSWORD '$MIMIC_PASSWORD';" + $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER_SQL; CREATE USER $MIMIC_USER_SQL WITH PASSWORD '$MIMIC_PASSWORD_SQL';" fi if [ "$MIMIC_DB" != "postgres" ]; then # drop and recreate the database - $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB;" - $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB OWNER $MIMIC_USER;" -fi \ No newline at end of file + $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB_SQL;" + $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB_SQL OWNER $MIMIC_USER_SQL;" +fi From d4a83056737ed70ea2bacc5776d95796cb217828 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:24:47 +0300 Subject: [PATCH 03/49] fix: pin sqlite import dtypes for mixed columns Stop pandas DtypeWarning on chartevents/datetimeevents/ inputevents_cv/noteevents during chunked import (#1237). Co-authored-by: Cursor --- mimic-iii/buildmimic/sqlite/import.py | 44 +++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index 801651c7..dc898b14 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -10,6 +10,34 @@ CHUNKSIZE = 10 ** 6 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME) +# Column dtypes for tables that trigger pandas mixed-type warnings when +# loaded with the default low_memory chunked inference (see #1237). +# Types mirror mimic-iii/buildmimic/postgres/postgres_create_tables.sql. +TABLE_DTYPES = { + "chartevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "datetimeevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "inputevents_cv": { + "ORIGINALRATE": "float64", + "ORIGINALRATEUOM": "string", + "ORIGINALSITE": "string", + }, + "noteevents": { + "CHARTTIME": "string", + "STORETIME": "string", + "ISERROR": "string", + }, +} + def _table_name_from_csv(filename: str) -> str: """Derive SQL table name from a CSV path (literal suffix, not str.strip).""" @@ -21,6 +49,18 @@ def _table_name_from_csv(filename: str) -> str: return name.lower() +def _read_csv(path, table, **kwargs): + # low_memory=False avoids dtype re-inference across chunks; table-specific + # dtypes pin the columns that otherwise flip between numeric/object. + return pd.read_csv( + path, + index_col="ROW_ID", + low_memory=False, + dtype=TABLE_DTYPES.get(table), + **kwargs, + ) + + if os.path.exists(DATABASE_NAME): msg = "File {} already exists.".format(DATABASE_NAME) print(msg) @@ -30,11 +70,11 @@ def _table_name_from_csv(filename: str) -> str: print("Starting processing {}".format(f)) table = _table_name_from_csv(f) if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, index_col="ROW_ID") + df = _read_csv(f, table) df.to_sql(table, CONNECTION_STRING) else: # If the file is too large, let's do the work in chunks - for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE): + for chunk in _read_csv(f, table, chunksize=CHUNKSIZE): chunk.to_sql(table, CONNECTION_STRING, if_exists="append") print("Finished processing {}".format(f)) From b03585805990a1d21d5776edfcce10eae696847d Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:29:02 +0300 Subject: [PATCH 04/49] fix: avoid escaped quote in neuroblock_dose stopped check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sqlglot rejects 'D/C''d' as adjacent string literals, breaking BigQuery→postgres/duckdb transpile of this concept. Co-authored-by: Cursor --- mimic-iii/concepts/durations/neuroblock_dose.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mimic-iii/concepts/durations/neuroblock_dose.sql b/mimic-iii/concepts/durations/neuroblock_dose.sql index 098b557e..3024d340 100644 --- a/mimic-iii/concepts/durations/neuroblock_dose.sql +++ b/mimic-iii/concepts/durations/neuroblock_dose.sql @@ -28,7 +28,7 @@ with drugmv as , 1 as drug -- the 'stopped' column indicates if a drug has been disconnected - , max(case when stopped in ('Stopped','D/C''d') then 1 else 0 end) as drug_stopped + , max(case when stopped in ('Stopped', CONCAT('D/C', CHR(39), 'd')) then 1 else 0 end) as drug_stopped -- we only include continuous infusions, therefore expect a rate , max(case @@ -68,7 +68,7 @@ with drugmv as , 1 as drug -- the 'stopped' column indicates if a drug has been disconnected - , max(case when stopped in ('Stopped','D/C''d') then 1 else 0 end) as drug_stopped + , max(case when stopped in ('Stopped', CONCAT('D/C', CHR(39), 'd')) then 1 else 0 end) as drug_stopped , max(case when valuenum <= 10 then 0 else 1 end) as drug_null -- educated guess! From 24c02b4623b057a34ce213a3f33c843b28c69ea3 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:46:57 +0300 Subject: [PATCH 05/49] fix: use DATETIME_DIFF for neuroblock gap check BigQuery rejects timestamp subtraction with interval compares; use DATETIME_DIFF(..., HOUR) like the rest of the concepts. Co-authored-by: Cursor --- mimic-iii/concepts/durations/neuroblock_dose.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mimic-iii/concepts/durations/neuroblock_dose.sql b/mimic-iii/concepts/durations/neuroblock_dose.sql index 3024d340..a1ce5c9a 100644 --- a/mimic-iii/concepts/durations/neuroblock_dose.sql +++ b/mimic-iii/concepts/durations/neuroblock_dose.sql @@ -179,7 +179,11 @@ select ) = 1 then 1 - when (CHARTTIME - (LAG(CHARTTIME, 1) OVER (partition by icustay_id, drug order by charttime))) > (interval '8 hours') then 1 + when DATETIME_DIFF( + CHARTTIME, + LAG(CHARTTIME, 1) OVER (partition by icustay_id, drug order by charttime), + HOUR + ) > 8 then 1 else null end as drug_start From 23f708457ef7a874f32adc4349837e38f9628447 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:53:32 +0300 Subject: [PATCH 06/49] fix: check $1 not PID in copy_concepts script $$1 expands to the shell PID, so the version arg was never validated. Co-authored-by: Cursor --- mimic-iv/concepts/copy_concepts_to_versioned_schema.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh b/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh index df0afa8f..547b64a4 100644 --- a/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh +++ b/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh @@ -1,6 +1,6 @@ #!/bin/bash # This script copies the concepts in the BigQuery table mimiciv_derived to mimiciv_${VERSION}_derived. -if [ -z "$$1" ]; then +if [ -z "$1" ]; then echo "Usage: $0 " exit 1 fi From cb18187432c059dc23c7a933af57d22a3c9e55fa Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 13:02:02 +0300 Subject: [PATCH 07/49] ci: dont fail sqlfluff job when annotate lacks fork perms Lint already ran; fork tokens cannot create check runs. Co-authored-by: Cursor --- .github/workflows/lint_sqlfluff.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint_sqlfluff.yml b/.github/workflows/lint_sqlfluff.yml index 9401198e..ca4c8c8b 100644 --- a/.github/workflows/lint_sqlfluff.yml +++ b/.github/workflows/lint_sqlfluff.yml @@ -35,8 +35,12 @@ jobs: shell: bash run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json - name: Annotate + # Fork PRs cannot create check runs with GITHUB_TOKEN; lint already ran. + continue-on-error: true + if: steps.get_files_to_lint.outputs.lintees != '' uses: yuzutech/annotations-action@v0.6.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" title: "SQLFluff Lint" - input: "./annotations.json" \ No newline at end of file + input: "./annotations.json" + ignore-missing-file: true \ No newline at end of file From e560823c2111a7c79b148eeacd8aedc98733c02f Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 13:35:16 +0300 Subject: [PATCH 08/49] fix: load .csv and .csv.gz in duckdb imports Co-authored-by: Cursor --- .../buildmimic/duckdb/import_duckdb.sh | 17 +++++++++++++--- .../buildmimic/duckdb/import_duckdb.sh | 17 +++++++++++++--- mimic-iv/buildmimic/duckdb/import_duckdb.sh | 20 ++++++++++++++++--- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh index 9142f7b4..ca408fdb 100644 --- a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh @@ -93,7 +93,11 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-ed @@ -102,9 +106,16 @@ find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do (ed) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .. " try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected ed/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh index c0425fe6..d9ae09a3 100644 --- a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh @@ -96,7 +96,11 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-note @@ -105,9 +109,10 @@ find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do (note) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .." OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -122,5 +127,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected note/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index ab1d37fa..23c58d9f 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -101,7 +101,13 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both uncompressed (.csv) and gzip (.csv.gz). The old '*.csv???' +# pattern only matched names with exactly three chars after ".csv" (i.e. .gz), +# so plain .csv files were silently skipped while the script still reported success. +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv @@ -110,9 +116,11 @@ find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do (hosp|icu) ;; # OK (*) continue; esac + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .. \c" OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -127,5 +135,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected hosp/ and icu/)." +fi +echo "Successfully finished loading data into $OUTFILE." From a2d01c4e6dd5b79395cc332813859c0013f951c7 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 13:35:16 +0300 Subject: [PATCH 09/49] fix: require epi/norepi rate > 0 for SOFA CV score 3 Co-authored-by: Cursor --- mimic-iii/concepts/severityscores/sofa.sql | 2 +- mimic-iv/concepts/firstday/first_day_sofa.sql | 4 ++-- mimic-iv/concepts/score/sofa.sql | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mimic-iii/concepts/severityscores/sofa.sql b/mimic-iii/concepts/severityscores/sofa.sql index 9afc5be1..52cb8d5f 100644 --- a/mimic-iii/concepts/severityscores/sofa.sql +++ b/mimic-iii/concepts/severityscores/sofa.sql @@ -218,7 +218,7 @@ left join `physionet-data.mimiciii_derived.gcs_first_day` gcs -- Cardiovascular , case when rate_dopamine > 15 or rate_epinephrine > 0.1 or rate_norepinephrine > 0.1 then 4 - when rate_dopamine > 5 or rate_epinephrine <= 0.1 or rate_norepinephrine <= 0.1 then 3 + when rate_dopamine > 5 or (rate_epinephrine > 0 and rate_epinephrine <= 0.1) or (rate_norepinephrine > 0 and rate_norepinephrine <= 0.1) then 3 when rate_dopamine > 0 or rate_dobutamine > 0 then 2 when meanbp_min < 70 then 1 when coalesce(meanbp_min, rate_dopamine, rate_dobutamine, rate_epinephrine, rate_norepinephrine) is null then null diff --git a/mimic-iv/concepts/firstday/first_day_sofa.sql b/mimic-iv/concepts/firstday/first_day_sofa.sql index 62282af7..ffd3d3c2 100644 --- a/mimic-iv/concepts/firstday/first_day_sofa.sql +++ b/mimic-iv/concepts/firstday/first_day_sofa.sql @@ -196,8 +196,8 @@ WITH vaso_stg AS ( OR rate_norepinephrine > 0.1 THEN 4 WHEN rate_dopamine > 5 - OR rate_epinephrine <= 0.1 - OR rate_norepinephrine <= 0.1 + OR (rate_epinephrine > 0 AND rate_epinephrine <= 0.1) + OR (rate_norepinephrine > 0 AND rate_norepinephrine <= 0.1) THEN 3 WHEN rate_dopamine > 0 OR rate_dobutamine > 0 THEN 2 WHEN mbp_min < 70 THEN 1 diff --git a/mimic-iv/concepts/score/sofa.sql b/mimic-iv/concepts/score/sofa.sql index 0dbf28d1..049b3920 100644 --- a/mimic-iv/concepts/score/sofa.sql +++ b/mimic-iv/concepts/score/sofa.sql @@ -280,8 +280,8 @@ WITH co AS ( OR rate_norepinephrine > 0.1 THEN 4 WHEN rate_dopamine > 5 - OR rate_epinephrine <= 0.1 - OR rate_norepinephrine <= 0.1 + OR (rate_epinephrine > 0 AND rate_epinephrine <= 0.1) + OR (rate_norepinephrine > 0 AND rate_norepinephrine <= 0.1) THEN 3 WHEN rate_dopamine > 0 OR rate_dobutamine > 0 From 1cfb063208b6a401ec7dab31b01974a1f917a3df Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 10:40:47 +0300 Subject: [PATCH 10/49] fix: quote datadir for postgres load and wget check already handles spaces (#481/#2068); load still used unquoted datadir and \cd without quoting, so paths with spaces broke at import. Co-authored-by: Cursor --- mimic-iii/buildmimic/postgres/Makefile | 8 ++++---- mimic-iii/buildmimic/postgres/postgres_load_data.sql | 2 +- mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql | 2 +- mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mimic-iii/buildmimic/postgres/Makefile b/mimic-iii/buildmimic/postgres/Makefile index 17ea42e2..29953adb 100644 --- a/mimic-iii/buildmimic/postgres/Makefile +++ b/mimic-iii/buildmimic/postgres/Makefile @@ -104,7 +104,7 @@ mimic-build-gz: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v mimic_data_dir=${datadir} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -151,7 +151,7 @@ mimic-build: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v mimic_data_dir=${DATADIR} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -184,7 +184,7 @@ ifeq ("$(physionetuser)","") @echo 'Call the makefile again with physionetuser=' @echo ' e.g. make eicu-download datadir=/path/to/data physionetuser=hello@physionet.org' else - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" endif mimic-demo-download: @@ -192,7 +192,7 @@ mimic-demo-download: @echo '-- Downloading MIMIC-III from PhysioNet --' @echo '------------------------------------------' @echo '' - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" #This is fairly inelegant and could be tidied with a for loop and an if to check for gzip, #but need to maintain compatibility with Windows, which baffling lacks these things diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data.sql b/mimic-iii/buildmimic/postgres/postgres_load_data.sql index 3fe94c5e..5245bac8 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql b/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql index c47799b5..36c27dca 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql b/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql index e993635e..4cf2835c 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; From 81e415508fdb19d38fd6303d86726e9b662abb5c Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 16:26:34 +0300 Subject: [PATCH 11/49] fix: write transpiled sql as utf-8 --- src/mimic_utils/transpile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mimic_utils/transpile.py b/src/mimic_utils/transpile.py index d4a65541..29924c18 100644 --- a/src/mimic_utils/transpile.py +++ b/src/mimic_utils/transpile.py @@ -113,7 +113,7 @@ def transpile_file( f"CREATE TABLE {derived_schema}{Path(source_file).stem} AS\n" ) + transpiled_query - with open(destination_file, "w") as write_file: + with open(destination_file, "w", encoding="utf-8") as write_file: write_file.write(transpiled_query) From b2711aa742540f4d2fe8939e9a165cebce9de69f Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 16:30:39 +0300 Subject: [PATCH 12/49] fix: exact table name match in validate_concepts.sh --- mimic-iv/concepts/validate_concepts.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mimic-iv/concepts/validate_concepts.sh b/mimic-iv/concepts/validate_concepts.sh index eeaddb82..645a82a3 100755 --- a/mimic-iv/concepts/validate_concepts.sh +++ b/mimic-iv/concepts/validate_concepts.sh @@ -32,7 +32,8 @@ fail=0 missing=0 while read -r tbl; do [ -z "${tbl}" ] && continue - if ! grep -q "^${tbl}," <<< "${ACTUAL}"; then + # exact field match (regex grep can false-hit prefixes) + if ! awk -F, -v t="${tbl}" '$1 == t { found=1 } END { exit !found }' <<< "${ACTUAL}"; then echo "MISSING: expected table ${DATASET}.${tbl} was not built" missing=1 fail=1 @@ -47,7 +48,7 @@ empty=0 while IFS=, read -r tbl rows; do [ -z "${tbl}" ] && continue # only validate the tables we actually build (ignore _metadata and any strays) - if grep -qx "${tbl}" <<< "${EXPECTED}" && [ "${rows}" -eq 0 ]; then + if grep -Fxq "${tbl}" <<< "${EXPECTED}" && [ "${rows:-}" -eq 0 ]; then echo "EMPTY: table ${DATASET}.${tbl} has 0 rows" empty=1 fail=1 From 8b8833a27a3c406d94cac2ae563f753a0efe0f12 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 13:55:05 +0300 Subject: [PATCH 13/49] fix: exclude ICD-10-CM C4A from Charlson malignancy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C4A (Merkel) sits in the C45–C58 string range; Quan 2005 uses WHO ICD-10 and excludes skin malignancy (#2017). Co-authored-by: Cursor --- mimic-iv/concepts/comorbidity/charlson.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mimic-iv/concepts/comorbidity/charlson.sql b/mimic-iv/concepts/comorbidity/charlson.sql index 15911b37..2546e7c8 100644 --- a/mimic-iv/concepts/comorbidity/charlson.sql +++ b/mimic-iv/concepts/comorbidity/charlson.sql @@ -285,6 +285,8 @@ WITH diag AS ( -- Any malignancy, including lymphoma and leukemia, -- except malignant neoplasm of skin. + -- Exclude ICD-10-CM-only C4A (Merkel); Quan 2005 is WHO ICD-10 (#2017). + -- C7A/C7B sit outside these ranges and are also left excluded. , MAX(CASE WHEN SUBSTR(icd9_code, 1, 3) BETWEEN '140' AND '172' OR @@ -302,7 +304,10 @@ WITH diag AS ( OR SUBSTR(icd10_code, 1, 3) BETWEEN 'C37' AND 'C41' OR - SUBSTR(icd10_code, 1, 3) BETWEEN 'C45' AND 'C58' + ( + SUBSTR(icd10_code, 1, 3) BETWEEN 'C45' AND 'C58' + AND SUBSTR(icd10_code, 1, 3) != 'C4A' + ) OR SUBSTR(icd10_code, 1, 3) BETWEEN 'C60' AND 'C76' OR From 3327400c1f51ef1ffca3287c2fe7f584d707bc20 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:08:41 +0300 Subject: [PATCH 14/49] fix: include provider/caregiver/ingredientevents in demo validate Demo loads these tables but validate_demo omitted them, so incomplete demo imports could still pass. Co-authored-by: Cursor --- mimic-iv/buildmimic/mysql/validate_demo.sql | 6 ++++++ mimic-iv/buildmimic/postgres/validate_demo.sql | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/mimic-iv/buildmimic/mysql/validate_demo.sql b/mimic-iv/buildmimic/mysql/validate_demo.sql index 575e8413..ebf4f02d 100644 --- a/mimic-iv/buildmimic/mysql/validate_demo.sql +++ b/mimic-iv/buildmimic/mysql/validate_demo.sql @@ -31,14 +31,17 @@ FROM ( SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) exp @@ -64,14 +67,17 @@ INNER JOIN SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM procedureevents ) obs diff --git a/mimic-iv/buildmimic/postgres/validate_demo.sql b/mimic-iv/buildmimic/postgres/validate_demo.sql index 303dd98e..049497e5 100644 --- a/mimic-iv/buildmimic/postgres/validate_demo.sql +++ b/mimic-iv/buildmimic/postgres/validate_demo.sql @@ -22,14 +22,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) @@ -54,14 +57,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM mimiciv_hosp.poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM mimiciv_hosp.prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM mimiciv_hosp.procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM mimiciv_hosp.provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM mimiciv_hosp.services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM mimiciv_hosp.transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM mimiciv_icu.icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM mimiciv_icu.caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM mimiciv_icu.d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.procedureevents ) From 475bbda9e62229159bcb4c9c77ac7c87dbad254d Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 12:36:07 +0300 Subject: [PATCH 15/49] fix: use globs instead of ls in make_concepts.sh --- mimic-iv/concepts/make_concepts.sh | 45 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/mimic-iv/concepts/make_concepts.sh b/mimic-iv/concepts/make_concepts.sh index 112bcdc2..cc633fc3 100644 --- a/mimic-iv/concepts/make_concepts.sh +++ b/mimic-iv/concepts/make_concepts.sh @@ -44,7 +44,7 @@ for table_path in demographics/icustay_times; do table=`echo $table_path | rev | cut -d/ -f1 | rev` echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done # generate tables in subfolders @@ -54,30 +54,29 @@ done # * organfailure depends on measurement for d in demographics comorbidity measurement medication organfailure treatment firstday score sepsis; do - for fn in `ls $d`; + for fn_path in "$d"/*.sql; do - # only run SQL queries - if [[ "${fn: -4}" == ".sql" ]]; then - # table name is file name minus extension - tbl=`echo $fn | rev | cut -d. -f2- | rev` + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") + # table name is file name minus extension + tbl="${fn%.sql}" - # skip certain tables where order matters - skip=0 - for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 - do - if [[ "${tbl}" == "${skip_table}" ]]; then - skip=1 - break - fi - done; - if [[ "${skip}" == "1" ]]; then - continue - fi - - # not skipping - so generate the table on bigquery - echo "Generating ${TARGET_DATASET}.${tbl}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${tbl} < ${d}/${fn} + # skip certain tables where order matters + skip=0 + for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 + do + if [[ "${tbl}" == "${skip_table}" ]]; then + skip=1 + break + fi + done; + if [[ "${skip}" == "1" ]]; then + continue fi + + # not skipping - so generate the table on bigquery + echo "Generating ${TARGET_DATASET}.${tbl}" + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${tbl}" < "${fn_path}" done done @@ -88,5 +87,5 @@ do table=`echo $table_path | rev | cut -d/ -f1 | rev` echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done From eafee6f5b3ceb5e1c3e9c4d9bb387cba2dda8c8c Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:08:41 +0300 Subject: [PATCH 16/49] fix: escape quotes in mimic-iii duckdb COPY paths Paths with apostrophes broke COPY string literals. Co-authored-by: Cursor --- mimic-iii/buildmimic/duckdb/import_duckdb.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mimic-iii/buildmimic/duckdb/import_duckdb.sh b/mimic-iii/buildmimic/duckdb/import_duckdb.sh index 15d4977f..7ccf9e2d 100755 --- a/mimic-iii/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iii/buildmimic/duckdb/import_duckdb.sh @@ -84,9 +84,11 @@ make_table_name () { # load data into database find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do make_table_name "$FILE" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .. \c" try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL echo "done!" done && echo "Successfully finished loading data into $OUTFILE." From 51bd9d439e1c4343ef92f6bf465b580afa808a60 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 11:22:20 +0300 Subject: [PATCH 17/49] fix: quote sqlite import.sh paths for spaces Co-authored-by: Cursor --- mimic-iii/buildmimic/sqlite/import.sh | 6 +++--- mimic-iv/buildmimic/sqlite/import.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mimic-iii/buildmimic/sqlite/import.sh b/mimic-iii/buildmimic/sqlite/import.sh index dbf6b9ce..58b85abb 100755 --- a/mimic-iii/buildmimic/sqlite/import.sh +++ b/mimic-iii/buildmimic/sqlite/import.sh @@ -28,11 +28,11 @@ for FILE in *; do TABLE_NAME=$(echo "${FILE%%.*}" | tr "[:upper:]" "[:lower:]") case "$FILE" in *csv) - IMPORT_CMD=".import $FILE $TABLE_NAME" + IMPORT_CMD=".import \"$FILE\" $TABLE_NAME" ;; # need to decompress csv before load *csv.gz) - IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME" + IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME" ;; # not a data file so skip *) @@ -40,7 +40,7 @@ for FILE in *; do ;; esac echo "Loading $FILE." - sqlite3 $OUTFILE < Date: Thu, 23 Jul 2026 14:15:02 +0300 Subject: [PATCH 18/49] fix: load plain .csv in mimic-iii sqlite import.py import.sh already accepts both; import.py only globbed *.csv.gz. Co-authored-by: Cursor --- mimic-iii/buildmimic/sqlite/import.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index dc898b14..4e0113d3 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -66,9 +66,16 @@ def _read_csv(path, table, **kwargs): print(msg) sys.exit() +# Prefer .csv.gz when both exist for the same table; also load plain .csv +# (import.sh already accepts both). +files_by_table = {} +for f in glob("*.csv"): + files_by_table[_table_name_from_csv(f)] = f for f in glob("*.csv.gz"): + files_by_table[_table_name_from_csv(f)] = f + +for table, f in sorted(files_by_table.items()): print("Starting processing {}".format(f)) - table = _table_name_from_csv(f) if os.path.getsize(f) < THRESHOLD_SIZE: df = _read_csv(f, table) df.to_sql(table, CONNECTION_STRING) From f7c8a45b413128cb5c16428952f548f8c49a27d1 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:15:02 +0300 Subject: [PATCH 19/49] fix: quote DIRNAME in duckdb import case patterns Unquoted case breaks when the parent folder name has spaces. Co-authored-by: Cursor --- mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh | 2 +- mimic-iv-note/buildmimic/duckdb/import_duckdb.sh | 2 +- mimic-iv/buildmimic/duckdb/import_duckdb.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh index ca408fdb..f6e6a2da 100644 --- a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh @@ -102,7 +102,7 @@ find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while # skip directories which we do not expect in mimic-iv-ed # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (ed) ;; # OK (*) continue; esac diff --git a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh index d9ae09a3..487d7405 100644 --- a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh @@ -105,7 +105,7 @@ find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while # skip directories which we do not expect in mimic-iv-note # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (note) ;; # OK (*) continue; esac diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index 23c58d9f..ee37c5a1 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -112,7 +112,7 @@ find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while # skip directories which we do not expect in mimic-iv # avoids syntax errors if mimic-iv-ed in the same dir - case $DIRNAME in + case "$DIRNAME" in (hosp|icu) ;; # OK (*) continue; esac From b3f8c6676dbc6fdb103bf2e0dddc2900ef7e8734 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 19:48:41 +0300 Subject: [PATCH 20/49] fix: quote paths in mimic-iii make-concepts.sh --- mimic-iii/concepts/make-concepts.sh | 140 ++++++++++++++-------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/mimic-iii/concepts/make-concepts.sh b/mimic-iii/concepts/make-concepts.sh index a7d1152c..01e15f00 100644 --- a/mimic-iii/concepts/make-concepts.sh +++ b/mimic-iii/concepts/make-concepts.sh @@ -14,108 +14,108 @@ echo '' set -x echo 'Top level files..' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.code_status < code_status.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.echo_data < echo_data.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.code_status" < "code_status.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.echo_data" < "echo_data.sql" echo 'Running queries in 10 directories.' echo 'Directory 1: demographics' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.heightweight < demographics/heightweight.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.icustay_detail < demographics/icustay_detail.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.heightweight" < "demographics/heightweight.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.icustay_detail" < "demographics/icustay_detail.sql" # Durations (usually of treatments) echo 'Directory 2: durations' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_classification < durations/ventilation_classification.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_durations < durations/ventilation_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.crrt_durations < durations/crrt_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.adenosine_durations < durations/adenosine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dobutamine_durations < durations/dobutamine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dopamine_durations < durations/dopamine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.epinephrine_durations < durations/epinephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.isuprel_durations < durations/isuprel_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.milrinone_durations < durations/milrinone_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.norepinephrine_durations < durations/norepinephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.phenylephrine_durations < durations/phenylephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressin_durations < durations/vasopressin_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressor_durations < durations/vasopressor_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.weight_durations < durations/weight_durations.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_classification" < "durations/ventilation_classification.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_durations" < "durations/ventilation_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.crrt_durations" < "durations/crrt_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.adenosine_durations" < "durations/adenosine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dobutamine_durations" < "durations/dobutamine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dopamine_durations" < "durations/dopamine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.epinephrine_durations" < "durations/epinephrine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.isuprel_durations" < "durations/isuprel_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.milrinone_durations" < "durations/milrinone_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.norepinephrine_durations" < "durations/norepinephrine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.phenylephrine_durations" < "durations/phenylephrine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vasopressin_durations" < "durations/vasopressin_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vasopressor_durations" < "durations/vasopressor_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.weight_durations" < "durations/weight_durations.sql" # dose queries for vasopressors -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dobutamine_dose < durations/dobutamine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dopamine_dose < durations/dopamine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.epinephrine_dose < durations/epinephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.norepinephrine_dose < durations/norepinephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.phenylephrine_dose < durations/phenylephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressin_dose < durations/vasopressin_dose.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dobutamine_dose" < "durations/dobutamine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dopamine_dose" < "durations/dopamine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.epinephrine_dose" < "durations/epinephrine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.norepinephrine_dose" < "durations/norepinephrine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.phenylephrine_dose" < "durations/phenylephrine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vasopressin_dose" < "durations/vasopressin_dose.sql" # "pivoted" tables which have icustay_id / timestamp as the primary key echo 'Directory 3: pivoted tables' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_vital < pivot/pivoted_vital.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_uo < pivot/pivoted_uo.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_rrt < pivot/pivoted_rrt.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_lab < pivot/pivoted_lab.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_invasive_lines < pivot/pivoted_invasive_lines.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_icp < pivot/pivoted_icp.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_height < pivot/pivoted_height.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_gcs < pivot/pivoted_gcs.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_fio2 < pivot/pivoted_fio2.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_bg < pivot/pivoted_bg.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_vital" < "pivot/pivoted_vital.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_uo" < "pivot/pivoted_uo.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_rrt" < "pivot/pivoted_rrt.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_lab" < "pivot/pivoted_lab.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_invasive_lines" < "pivot/pivoted_invasive_lines.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_icp" < "pivot/pivoted_icp.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_height" < "pivot/pivoted_height.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_gcs" < "pivot/pivoted_gcs.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_fio2" < "pivot/pivoted_fio2.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_bg" < "pivot/pivoted_bg.sql" # pivoted_bg_art must be run after pivoted_bg -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_bg_art < pivot/pivoted_bg_art.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_bg_art" < "pivot/pivoted_bg_art.sql" # pivoted oasis depends on icustay_hours in demographics -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_oasis < pivot/pivoted_oasis.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_oasis" < "pivot/pivoted_oasis.sql" # pivoted sofa depends on many above pivoted views, ventilation_durations, and dose queries -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_sofa < pivot/pivoted_sofa.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_sofa" < "pivot/pivoted_sofa.sql" echo 'Directory 4: comorbidity' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_ahrq_v37 < comorbidity/elixhauser_ahrq_v37.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_ahrq_v37_no_drg < comorbidity/elixhauser_ahrq_v37_no_drg.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_quan < comorbidity/elixhauser_quan.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_score_ahrq < comorbidity/elixhauser_score_ahrq.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_score_quan < comorbidity/elixhauser_score_quan.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_ahrq_v37" < "comorbidity/elixhauser_ahrq_v37.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_ahrq_v37_no_drg" < "comorbidity/elixhauser_ahrq_v37_no_drg.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_quan" < "comorbidity/elixhauser_quan.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_score_ahrq" < "comorbidity/elixhauser_score_ahrq.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_score_quan" < "comorbidity/elixhauser_score_quan.sql" echo 'Directory 5: firstday' # data which is extracted from a patient's first ICU stay -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.blood_gas_first_day < firstday/blood_gas_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.blood_gas_first_day_arterial < firstday/blood_gas_first_day_arterial.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.gcs_first_day < firstday/gcs_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.labs_first_day < firstday/labs_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.rrt_first_day < firstday/rrt_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.urine_output_first_day < firstday/urine_output_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_first_day < firstday/ventilation_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vitals_first_day < firstday/vitals_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.weight_first_day < firstday/weight_first_day.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.blood_gas_first_day" < "firstday/blood_gas_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.blood_gas_first_day_arterial" < "firstday/blood_gas_first_day_arterial.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.gcs_first_day" < "firstday/gcs_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.labs_first_day" < "firstday/labs_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.rrt_first_day" < "firstday/rrt_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.urine_output_first_day" < "firstday/urine_output_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_first_day" < "firstday/ventilation_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vitals_first_day" < "firstday/vitals_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.weight_first_day" < "firstday/weight_first_day.sql" echo 'Directory 6: fluid_balance' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.urine_output < fluid_balance/urine_output.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.urine_output" < "fluid_balance/urine_output.sql" echo 'Directory 7: sepsis' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.angus < sepsis/angus.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.martin < sepsis/martin.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.explicit < sepsis/explicit.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.angus" < "sepsis/angus.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.martin" < "sepsis/martin.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.explicit" < "sepsis/explicit.sql" # diagnosis mapping using CCS echo 'Directory 8: diagnosis' # load the ccs_multi_dx.csv.gz file into bq -bq load --source_format=CSV ${TARGET_DATASET}.ccs_multi_dx diagnosis/ccs_multi_dx.csv.gz diagnosis/ccs_multi_dx.json -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ccs_dx < diagnosis/ccs_dx.sql +bq load --source_format=CSV "${TARGET_DATASET}.ccs_multi_dx" "diagnosis/ccs_multi_dx.csv.gz" "diagnosis/ccs_multi_dx.json" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ccs_dx" < "diagnosis/ccs_dx.sql" # Organ failure scores echo 'Directory 9: organfailure' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_creatinine < organfailure/kdigo_creatinine.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_uo < organfailure/kdigo_uo.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages < organfailure/kdigo_stages.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages_7day < organfailure/kdigo_stages_7day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages_48hr < organfailure/kdigo_stages_48hr.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.meld < organfailure/meld.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_creatinine" < "organfailure/kdigo_creatinine.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_uo" < "organfailure/kdigo_uo.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_stages" < "organfailure/kdigo_stages.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_stages_7day" < "organfailure/kdigo_stages_7day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_stages_48hr" < "organfailure/kdigo_stages_48hr.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.meld" < "organfailure/meld.sql" # Severity of illness scores (requires many views from above) echo 'Directory 10: severityscores' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.oasis < severityscores/oasis.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sofa < severityscores/sofa.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.saps < severityscores/saps.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sapsii < severityscores/sapsii.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.apsiii < severityscores/apsiii.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.lods < severityscores/lods.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sirs < severityscores/sirs.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.oasis" < "severityscores/oasis.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.sofa" < "severityscores/sofa.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.saps" < "severityscores/saps.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.sapsii" < "severityscores/sapsii.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.apsiii" < "severityscores/apsiii.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.lods" < "severityscores/lods.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.sirs" < "severityscores/sirs.sql" echo 'Finished creating concepts.' \ No newline at end of file From dd56cd399054b801abf391778c213c96e62114c5 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 11:54:23 +0300 Subject: [PATCH 21/49] fix: low_memory=False on mimic-iv sqlite read_csv --- mimic-iv/buildmimic/sqlite/import.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mimic-iv/buildmimic/sqlite/import.py b/mimic-iv/buildmimic/sqlite/import.py index 68d43647..31b880c4 100644 --- a/mimic-iv/buildmimic/sqlite/import.py +++ b/mimic-iv/buildmimic/sqlite/import.py @@ -110,7 +110,7 @@ def main(): if args.limit > 0: for f in data_files: if 'patients' in f.name: - pt = pd.read_csv(f) + pt = pd.read_csv(f, low_memory=False) break if pt is None: raise FileNotFoundError('Unable to find a patients file in current folder.') @@ -158,7 +158,7 @@ def main(): tablename = tablenames[i] print("Starting processing {}".format(tablename), end='.. ') if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, dtype=mimic_dtypes) + df = pd.read_csv(f, dtype=mimic_dtypes, low_memory=False) df = process_dataframe(df, subjects=subjects) df.to_sql(tablename, connection, index=False) row_counts[tablename] += len(df) From 6f9950994b4ac0f642ddde612b8b23cb9fb31ef9 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:19:38 +0300 Subject: [PATCH 22/49] fix: use printf for duckdb load progress echo .. \c prints a literal \c on bash; printf is portable. Co-authored-by: Cursor --- mimic-iii/buildmimic/duckdb/import_duckdb.sh | 2 +- mimic-iv/buildmimic/duckdb/import_duckdb.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mimic-iii/buildmimic/duckdb/import_duckdb.sh b/mimic-iii/buildmimic/duckdb/import_duckdb.sh index 7ccf9e2d..badf83b8 100755 --- a/mimic-iii/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iii/buildmimic/duckdb/import_duckdb.sh @@ -86,7 +86,7 @@ find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do make_table_name "$FILE" # Escape single quotes for SQL string literal FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") - echo "Loading $FILE .. \c" + printf "Loading %s .. " "$FILE" try duckdb "$OUTFILE" <<-EOSQL COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index ee37c5a1..32f608c4 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -118,7 +118,7 @@ find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while esac # Escape single quotes for SQL string literal FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") - echo "Loading $FILE .. \c" + printf "Loading %s .. " "$FILE" OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL From f8590e83653bb9b807b2239536f4785de8d405f0 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 16:21:30 +0300 Subject: [PATCH 23/49] fix: use globs and quote paths in cxr label scripts --- mimic-iv-cxr/txt/negbio/run_negbio.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mimic-iv-cxr/txt/negbio/run_negbio.sh b/mimic-iv-cxr/txt/negbio/run_negbio.sh index 13cba479..d58e7fbd 100755 --- a/mimic-iv-cxr/txt/negbio/run_negbio.sh +++ b/mimic-iv-cxr/txt/negbio/run_negbio.sh @@ -32,20 +32,22 @@ sleep 2 # mimic_cxr_001.csv # mimic_cxr_002.csv # .. etc -for fn in `ls $BASE_FOLDER`; +for fn_path in "$BASE_FOLDER"/mimic_cxr_*.csv; do + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") echo "Looping through files with mimic_cxr_###.csv pattern." # validate it's a mimic_cxr sections file if [[ $fn =~ ^mimic_cxr_[0-9]+.csv$ ]]; then - export INPUT_FILE=${BASE_FOLDER}/$fn + export INPUT_FILE=$fn_path # remove extension from filename # sets the folder location to stem of original filename # all intermediate files will be saved in this folder export OUTPUT_DIR=${INPUT_FILE::-4} echo $OUTPUT_DIR - running NegBio.. - python $NEGBIO_PATH/negbio/negbio_csv2bioc.py --output $OUTPUT_DIR/report $INPUT_FILE + python "$NEGBIO_PATH/negbio/negbio_csv2bioc.py" --output "$OUTPUT_DIR/report" "$INPUT_FILE" python $NEGBIO_PATH/negbio/negbio_pipeline.py section_split --pattern $NEGBIO_PATH/patterns/section_titles_cxr8.txt --output $OUTPUT_DIR/sections $OUTPUT_DIR/report/* --workers=6 python $NEGBIO_PATH/negbio/negbio_pipeline.py ssplit --output $OUTPUT_DIR/ssplit $OUTPUT_DIR/sections/* --workers=6 python $NEGBIO_PATH/negbio/negbio_pipeline.py parse --output $OUTPUT_DIR/parse $OUTPUT_DIR/ssplit/* --workers=6 From cde84eb873dd624ac5b9ce5751c3fb4751fc7eb9 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 16:21:51 +0300 Subject: [PATCH 24/49] fix: glob + quote paths in chexpert runner too --- .../txt/chexpert/run_chexpert_on_files.sh | 21 +++++++++++++------ mimic-iv-cxr/txt/negbio/run_negbio.sh | 4 ++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh index 52e34e0f..f996541a 100644 --- a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh +++ b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh @@ -22,11 +22,20 @@ fi sleep 2 # loop through each .csv file in the section folder -for fn in `ls $REPORT_PATH`; do - echo `date`: $fn - fn_stem=`echo $fn | cut -d. -f 1` +for fn_path in "$REPORT_PATH"/*.csv; do + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") + echo "$(date): $fn" + fn_stem=${fn%.csv} # run chexpert - must be run from chexpert folder - python $CHEXPERT_PATH/label.py --verbose --reports_path $REPORT_PATH/$fn --output_path ${fn_stem}_labeled.csv --mention_phrases_dir $CHEXPERT_PATH/phrases/mention --unmention_phrases_dir $CHEXPERT_PATH/phrases/unmention --pre_negation_uncertainty_path $CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt --negation_path $CHEXPERT_PATH/patterns/negation.txt --post_negation_uncertainty_path $CHEXPERT_PATH/patterns/post_negation_uncertainty.txt - echo `date`: done! + python "$CHEXPERT_PATH/label.py" --verbose \ + --reports_path "$fn_path" \ + --output_path "${fn_stem}_labeled.csv" \ + --mention_phrases_dir "$CHEXPERT_PATH/phrases/mention" \ + --unmention_phrases_dir "$CHEXPERT_PATH/phrases/unmention" \ + --pre_negation_uncertainty_path "$CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt" \ + --negation_path "$CHEXPERT_PATH/patterns/negation.txt" \ + --post_negation_uncertainty_path "$CHEXPERT_PATH/patterns/post_negation_uncertainty.txt" + echo "$(date): done!" echo '' -done \ No newline at end of file +done diff --git a/mimic-iv-cxr/txt/negbio/run_negbio.sh b/mimic-iv-cxr/txt/negbio/run_negbio.sh index d58e7fbd..dab200cd 100755 --- a/mimic-iv-cxr/txt/negbio/run_negbio.sh +++ b/mimic-iv-cxr/txt/negbio/run_negbio.sh @@ -46,7 +46,7 @@ do # all intermediate files will be saved in this folder export OUTPUT_DIR=${INPUT_FILE::-4} - echo $OUTPUT_DIR - running NegBio.. + echo "$OUTPUT_DIR - running NegBio.." python "$NEGBIO_PATH/negbio/negbio_csv2bioc.py" --output "$OUTPUT_DIR/report" "$INPUT_FILE" python $NEGBIO_PATH/negbio/negbio_pipeline.py section_split --pattern $NEGBIO_PATH/patterns/section_titles_cxr8.txt --output $OUTPUT_DIR/sections $OUTPUT_DIR/report/* --workers=6 python $NEGBIO_PATH/negbio/negbio_pipeline.py ssplit --output $OUTPUT_DIR/ssplit $OUTPUT_DIR/sections/* --workers=6 @@ -57,7 +57,7 @@ do # ultimate filename we save the labels to export OUTPUT_LABELS=$OUTPUT_DIR/${fn::-4}_labels.csv - python $NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py --phrases_file $NEGBIO_PATH/patterns/chexpert_phrases.yml --output $OUTPUT_LABELS $OUTPUT_DIR/neg/* + python "$NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py" --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_LABELS" "$OUTPUT_DIR"/neg/* fi done echo "Done looping through files." From e7d5561d7ddb0f56cd5d7f86667d7c2c327d78ac Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:26:00 +0300 Subject: [PATCH 25/49] fix: quote mimic_data_dir for postgres \\cd in IV/ED/Note Same path-with-spaces break as #481; use :'mimic_data_dir'. Co-authored-by: Cursor --- mimic-iv-ed/buildmimic/postgres/load.sql | 2 +- mimic-iv-ed/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv-ed/buildmimic/postgres/load_gz.sql | 2 +- mimic-iv-note/buildmimic/postgres/load.sql | 2 +- mimic-iv-note/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv-note/buildmimic/postgres/load_gz.sql | 2 +- mimic-iv/buildmimic/postgres/load.sql | 2 +- mimic-iv/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv/buildmimic/postgres/load_gz.sql | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mimic-iv-ed/buildmimic/postgres/load.sql b/mimic-iv-ed/buildmimic/postgres/load.sql index 81fc9b2a..5b7e50ca 100644 --- a/mimic-iv-ed/buildmimic/postgres/load.sql +++ b/mimic-iv-ed/buildmimic/postgres/load.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load.sql -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-ed/buildmimic/postgres/load_7z.sql b/mimic-iv-ed/buildmimic/postgres/load_7z.sql index 15d031f9..b5473361 100644 --- a/mimic-iv-ed/buildmimic/postgres/load_7z.sql +++ b/mimic-iv-ed/buildmimic/postgres/load_7z.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load_7z.sql -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-ed/buildmimic/postgres/load_gz.sql b/mimic-iv-ed/buildmimic/postgres/load_gz.sql index 6e92a132..0d42e5d1 100644 --- a/mimic-iv-ed/buildmimic/postgres/load_gz.sql +++ b/mimic-iv-ed/buildmimic/postgres/load_gz.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-note/buildmimic/postgres/load.sql b/mimic-iv-note/buildmimic/postgres/load.sql index 38a79a82..27e73761 100644 --- a/mimic-iv-note/buildmimic/postgres/load.sql +++ b/mimic-iv-note/buildmimic/postgres/load.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv-note/buildmimic/postgres/load_7z.sql b/mimic-iv-note/buildmimic/postgres/load_7z.sql index 57c4dee6..2d02f598 100644 --- a/mimic-iv-note/buildmimic/postgres/load_7z.sql +++ b/mimic-iv-note/buildmimic/postgres/load_7z.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv-note/buildmimic/postgres/load_gz.sql b/mimic-iv-note/buildmimic/postgres/load_gz.sql index bbe9ba64..8b32d85d 100644 --- a/mimic-iv-note/buildmimic/postgres/load_gz.sql +++ b/mimic-iv-note/buildmimic/postgres/load_gz.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load.sql b/mimic-iv/buildmimic/postgres/load.sql index e48848d9..c2c2f52f 100644 --- a/mimic-iv/buildmimic/postgres/load.sql +++ b/mimic-iv/buildmimic/postgres/load.sql @@ -5,7 +5,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load.sql -- The script assumes the files are in the hosp and icu subfolders of mimic_data_dir -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure correct encoding is defined as -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load_7z.sql b/mimic-iv/buildmimic/postgres/load_7z.sql index 5f238bc4..8ccc4604 100644 --- a/mimic-iv/buildmimic/postgres/load_7z.sql +++ b/mimic-iv/buildmimic/postgres/load_7z.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load_gz.sql b/mimic-iv/buildmimic/postgres/load_gz.sql index f77ac82f..e9a895a4 100644 --- a/mimic-iv/buildmimic/postgres/load_gz.sql +++ b/mimic-iv/buildmimic/postgres/load_gz.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; From 6b3785640cc87ca8f36b65a689e7b0eb3d37f2bf Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:30:20 +0300 Subject: [PATCH 26/49] revert: unquote psql \\cd mimic_data_dir :'var' makes \\cd include literal quotes in the path and breaks CI. Co-authored-by: Cursor --- mimic-iii/buildmimic/postgres/postgres_load_data.sql | 2 +- mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql | 2 +- mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql | 2 +- mimic-iv-ed/buildmimic/postgres/load.sql | 2 +- mimic-iv-ed/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv-ed/buildmimic/postgres/load_gz.sql | 2 +- mimic-iv-note/buildmimic/postgres/load.sql | 2 +- mimic-iv-note/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv-note/buildmimic/postgres/load_gz.sql | 2 +- mimic-iv/buildmimic/postgres/load.sql | 2 +- mimic-iv/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv/buildmimic/postgres/load_gz.sql | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data.sql b/mimic-iii/buildmimic/postgres/postgres_load_data.sql index 5245bac8..3fe94c5e 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql b/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql index 36c27dca..c47799b5 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql b/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql index 4cf2835c..e993635e 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; diff --git a/mimic-iv-ed/buildmimic/postgres/load.sql b/mimic-iv-ed/buildmimic/postgres/load.sql index 5b7e50ca..81fc9b2a 100644 --- a/mimic-iv-ed/buildmimic/postgres/load.sql +++ b/mimic-iv-ed/buildmimic/postgres/load.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load.sql -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-ed/buildmimic/postgres/load_7z.sql b/mimic-iv-ed/buildmimic/postgres/load_7z.sql index b5473361..15d031f9 100644 --- a/mimic-iv-ed/buildmimic/postgres/load_7z.sql +++ b/mimic-iv-ed/buildmimic/postgres/load_7z.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load_7z.sql -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-ed/buildmimic/postgres/load_gz.sql b/mimic-iv-ed/buildmimic/postgres/load_gz.sql index 0d42e5d1..6e92a132 100644 --- a/mimic-iv-ed/buildmimic/postgres/load_gz.sql +++ b/mimic-iv-ed/buildmimic/postgres/load_gz.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-note/buildmimic/postgres/load.sql b/mimic-iv-note/buildmimic/postgres/load.sql index 27e73761..38a79a82 100644 --- a/mimic-iv-note/buildmimic/postgres/load.sql +++ b/mimic-iv-note/buildmimic/postgres/load.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv-note/buildmimic/postgres/load_7z.sql b/mimic-iv-note/buildmimic/postgres/load_7z.sql index 2d02f598..57c4dee6 100644 --- a/mimic-iv-note/buildmimic/postgres/load_7z.sql +++ b/mimic-iv-note/buildmimic/postgres/load_7z.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv-note/buildmimic/postgres/load_gz.sql b/mimic-iv-note/buildmimic/postgres/load_gz.sql index 8b32d85d..bbe9ba64 100644 --- a/mimic-iv-note/buildmimic/postgres/load_gz.sql +++ b/mimic-iv-note/buildmimic/postgres/load_gz.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load.sql b/mimic-iv/buildmimic/postgres/load.sql index c2c2f52f..e48848d9 100644 --- a/mimic-iv/buildmimic/postgres/load.sql +++ b/mimic-iv/buildmimic/postgres/load.sql @@ -5,7 +5,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load.sql -- The script assumes the files are in the hosp and icu subfolders of mimic_data_dir -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure correct encoding is defined as -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load_7z.sql b/mimic-iv/buildmimic/postgres/load_7z.sql index 8ccc4604..5f238bc4 100644 --- a/mimic-iv/buildmimic/postgres/load_7z.sql +++ b/mimic-iv/buildmimic/postgres/load_7z.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load_gz.sql b/mimic-iv/buildmimic/postgres/load_gz.sql index e9a895a4..f77ac82f 100644 --- a/mimic-iv/buildmimic/postgres/load_gz.sql +++ b/mimic-iv/buildmimic/postgres/load_gz.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; From 1bb1e339a06bf946388ed41701943208937cac8f Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:45:21 +0300 Subject: [PATCH 27/49] fix: require epi/norepi rate > 0 in pivoted_sofa CV score Same SOFA cardiovascular false-positive as sofa.sql when rates are null. Co-authored-by: Cursor --- mimic-iii/concepts/pivot/pivoted_sofa.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mimic-iii/concepts/pivot/pivoted_sofa.sql b/mimic-iii/concepts/pivot/pivoted_sofa.sql index 6d4940fd..c7947368 100644 --- a/mimic-iii/concepts/pivot/pivoted_sofa.sql +++ b/mimic-iii/concepts/pivot/pivoted_sofa.sql @@ -228,7 +228,7 @@ WITH co AS -- Cardiovascular , cast(case when rate_dopamine > 15 or rate_epinephrine > 0.1 or rate_norepinephrine > 0.1 then 4 - when rate_dopamine > 5 or rate_epinephrine <= 0.1 or rate_norepinephrine <= 0.1 then 3 + when rate_dopamine > 5 or (rate_epinephrine > 0 and rate_epinephrine <= 0.1) or (rate_norepinephrine > 0 and rate_norepinephrine <= 0.1) then 3 when rate_dopamine > 0 or rate_dobutamine > 0 then 2 when meanbp_min < 70 then 1 when coalesce(meanbp_min, rate_dopamine, rate_dobutamine, rate_epinephrine, rate_norepinephrine) is null then null From 2d4f3ae0abbd79c24774d1184e6f4c99acb6ed19 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:45:21 +0300 Subject: [PATCH 28/49] fix: keep DuckDB GENERATE_ARRAY as a list for UNNEST Bare GENERATE_SERIES broke icustay_hourly on older DuckDB (#1736). Co-authored-by: Cursor --- .../concepts_duckdb/demographics/icustay_hours.sql | 2 +- .../concepts_duckdb/demographics/icustay_hourly.sql | 2 +- src/mimic_utils/sqlglot_dialects/duckdb.py | 12 ++++++++++++ tests/test_transpile.py | 3 +++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql b/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql index 7b459f3e..12624abf 100644 --- a/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql +++ b/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql @@ -4,7 +4,7 @@ WITH all_hours AS ( SELECT it.icustay_id, STRPTIME(STRFTIME(CAST(it.intime_hr + INTERVAL '59' MINUTE AS TIMESTAMP), '%Y-%m-%d %H:00:00'), '%Y-%m-%d %H:00:00') AS endtime, - GENERATE_SERIES(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS BIGINT)) AS hrs + (SELECT list(g) FROM generate_series(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS BIGINT)) AS t(g)) AS hrs FROM mimiciii_derived.icustay_times AS it ) SELECT diff --git a/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql b/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql index 9341d3a4..682a6f5c 100644 --- a/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql +++ b/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql @@ -8,7 +8,7 @@ WITH all_hours AS ( THEN it.intime_hr ELSE DATE_TRUNC('HOUR', CAST(it.intime_hr AS TIMESTAMP)) + INTERVAL '1' HOUR END AS endtime, - GENERATE_SERIES(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS INT)) AS hrs + (SELECT list(g) FROM generate_series(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS INT)) AS t(g)) AS hrs FROM mimiciv_derived.icustay_times AS it ) SELECT diff --git a/src/mimic_utils/sqlglot_dialects/duckdb.py b/src/mimic_utils/sqlglot_dialects/duckdb.py index 12e83bfd..2c3262b9 100644 --- a/src/mimic_utils/sqlglot_dialects/duckdb.py +++ b/src/mimic_utils/sqlglot_dialects/duckdb.py @@ -3,6 +3,10 @@ - ``NUMERIC`` is ``DECIMAL(38, 9)``, but DuckDB's bare ``DECIMAL`` defaults to ``DECIMAL(18, 3)``, which silently rounds values to three decimal places (e.g. ``CAST(0.0255 AS NUMERIC)`` becomes ``0.026`` before any explicit ``ROUND``). +- ``GENERATE_ARRAY`` must remain a LIST (for later ``UNNEST``). Native sqlglot +emits ``GENERATE_SERIES``, which is a set-returning function; wrapping as a +list matches Postgres ``ARRAY(GENERATE_SERIES)`` and keeps ``UNNEST(hrs)`` +reliable across DuckDB versions (#1736). """ import re @@ -41,6 +45,13 @@ def _parse_datetime_sql(self: DuckDB.Generator, expression: exp.ParseDatetime) - return f"STRPTIME({this}, {fmt})" +# GENERATE_ARRAY(a, b) -> list via generate_series (inclusive), for UNNEST. +def _generate_array_sql(self: DuckDB.Generator, expression: exp.Expression) -> str: + start = self.sql(expression, "start") + end = self.sql(expression, "end") + return f"(SELECT list(g) FROM generate_series({start}, {end}) AS t(g))" + + class MimicDuckDB(DuckDB): class Generator(DuckDB.Generator): def datatype_sql(self, expression: exp.DataType) -> str: @@ -54,4 +65,5 @@ def datatype_sql(self, expression: exp.DataType) -> str: **DuckDB.Generator.TRANSFORMS, exp.RegexpExtract: _regexp_extract_sql, exp.ParseDatetime: _parse_datetime_sql, + exp.GenerateSeries: _generate_array_sql, } diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 952240bc..8e18bad3 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -71,6 +71,9 @@ def t(bq: str, dialect: str) -> str: # GENERATE_ARRAY -> ARRAY(SELECT ... GENERATE_SERIES) (postgres) ("generate_array_pg", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "postgres", "SELECT ARRAY(SELECT * FROM GENERATE_SERIES(-24, 5)) AS hrs FROM t"), + # GENERATE_ARRAY -> list via generate_series (duckdb); must stay list for UNNEST + ("generate_array_duckdb", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "duckdb", + "SELECT (SELECT list(g) FROM generate_series(-24, 5) AS t(g)) AS hrs FROM t"), # handled natively by sqlglot 30.x (regression guards) ("datetime_date_pg", "SELECT DATETIME(me.chartdate) FROM t me", "postgres", From a229b37e0944f5cf45f9526e85ed10b7886ea795 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:52:09 +0300 Subject: [PATCH 29/49] fix: include CCO and Arctic Sun temps in vitalsign Itemids 226329/227632/227634 were omitted from temperature (#1358). Co-authored-by: Cursor --- mimic-iv/concepts/measurement/vitalsign.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mimic-iv/concepts/measurement/vitalsign.sql b/mimic-iv/concepts/measurement/vitalsign.sql index 030e4055..c2899b4d 100644 --- a/mimic-iv/concepts/measurement/vitalsign.sql +++ b/mimic-iv/concepts/measurement/vitalsign.sql @@ -53,7 +53,7 @@ SELECT AND valuenum < 120 THEN (valuenum - 32) / 1.8 -- already in degC, no conversion necessary - WHEN itemid IN (223762) + WHEN itemid IN (223762, 226329, 227632, 227634) AND valuenum > 10 AND valuenum < 50 THEN valuenum END) @@ -91,7 +91,9 @@ WHERE ce.stay_id IS NOT NULL , 220621 -- Glucose (serum) , 226537 -- Glucose (whole blood) -- TEMPERATURE - -- 226329 -- Blood Temperature CCO (C) + , 226329 -- Blood Temperature CCO (C) + , 227632 -- Arctic Sun/Alsius Temp # 1 C + , 227634 -- Arctic Sun/Alsius Temp # 2 C , 223762 -- "Temperature Celsius" , 223761 -- "Temperature Fahrenheit" , 224642 -- Temperature Site From be93cfab00df02a745cc89384ffdc4d8071232d5 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:52:09 +0300 Subject: [PATCH 30/49] fix: Charlson age_score uses half-open decade bands Age 50 was scored 0; CCI awards 1 point starting at 50. Co-authored-by: Cursor --- mimic-iv/concepts/comorbidity/charlson.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mimic-iv/concepts/comorbidity/charlson.sql b/mimic-iv/concepts/comorbidity/charlson.sql index 2546e7c8..cb6c168c 100644 --- a/mimic-iv/concepts/comorbidity/charlson.sql +++ b/mimic-iv/concepts/comorbidity/charlson.sql @@ -355,10 +355,11 @@ WITH diag AS ( SELECT hadm_id , age - , CASE WHEN age <= 50 THEN 0 - WHEN age <= 60 THEN 1 - WHEN age <= 70 THEN 2 - WHEN age <= 80 THEN 3 + -- CCI age bands are [50,60), [60,70), [70,80), 80+ (not <= decade). + , CASE WHEN age < 50 THEN 0 + WHEN age < 60 THEN 1 + WHEN age < 70 THEN 2 + WHEN age < 80 THEN 3 ELSE 4 END AS age_score FROM `physionet-data.mimiciv_derived.age` ) From aa06aea92db92840baa2c1a4d05770d9012b12bb Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:52:09 +0300 Subject: [PATCH 31/49] fix: keep o2-device-only rows after FULL OUTER JOIN WHERE ce.rn = 1 dropped charttimes with devices but no flow. Co-authored-by: Cursor --- mimic-iv/concepts/measurement/oxygen_delivery.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mimic-iv/concepts/measurement/oxygen_delivery.sql b/mimic-iv/concepts/measurement/oxygen_delivery.sql index e76f6303..9c582c36 100644 --- a/mimic-iv/concepts/measurement/oxygen_delivery.sql +++ b/mimic-iv/concepts/measurement/oxygen_delivery.sql @@ -89,8 +89,9 @@ WITH ce_stg1 AS ( FULL OUTER JOIN o2 ON ce.subject_id = o2.subject_id AND ce.charttime = o2.charttime - -- limit to 1 row per subject_id/charttime/itemid from ce_stg2 - WHERE ce.rn = 1 + -- limit to 1 row per subject_id/charttime/itemid from ce_stg2; + -- keep o2-only charttimes (ce side NULL after the full outer join) + WHERE ce.rn = 1 OR ce.subject_id IS NULL ) SELECT From 8f2b9c816ec5c4727b88d56bb95b24a81efbc52e Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:01:28 +0300 Subject: [PATCH 32/49] fix: exclude MetaVision pause gaps from vaso durations Grouping by linkorderid with min/max start/end treated pauses as continuous infusion (#1808). Keep per-row intervals and merge only overlapping ones. Co-authored-by: Cursor --- .../durations/adenosine_durations.sql | 34 +++++++++++++++++-- .../durations/dobutamine_durations.sql | 34 +++++++++++++++++-- .../concepts/durations/dopamine_durations.sql | 34 +++++++++++++++++-- .../durations/epinephrine_durations.sql | 34 +++++++++++++++++-- .../concepts/durations/isuprel_durations.sql | 34 +++++++++++++++++-- .../durations/milrinone_durations.sql | 34 +++++++++++++++++-- .../durations/norepinephrine_durations.sql | 34 +++++++++++++++++-- .../durations/phenylephrine_durations.sql | 34 +++++++++++++++++-- .../durations/vasopressin_durations.sql | 34 +++++++++++++++++-- .../durations/vasopressor_durations.sql | 5 +-- 10 files changed, 282 insertions(+), 29 deletions(-) diff --git a/mimic-iii/concepts/durations/adenosine_durations.sql b/mimic-iii/concepts/durations/adenosine_durations.sql index 4bd2afcd..8859198a 100644 --- a/mimic-iii/concepts/durations/adenosine_durations.sql +++ b/mimic-iii/concepts/durations/adenosine_durations.sql @@ -198,11 +198,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221282 -- adenosine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/dobutamine_durations.sql b/mimic-iii/concepts/durations/dobutamine_durations.sql index bdac0706..7063348c 100644 --- a/mimic-iii/concepts/durations/dobutamine_durations.sql +++ b/mimic-iii/concepts/durations/dobutamine_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221653 -- dobutamine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/dopamine_durations.sql b/mimic-iii/concepts/durations/dopamine_durations.sql index 65ccea38..1f5672ce 100644 --- a/mimic-iii/concepts/durations/dopamine_durations.sql +++ b/mimic-iii/concepts/durations/dopamine_durations.sql @@ -198,11 +198,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221662 -- dopamine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/epinephrine_durations.sql b/mimic-iii/concepts/durations/epinephrine_durations.sql index a51a4d55..43cfae00 100644 --- a/mimic-iii/concepts/durations/epinephrine_durations.sql +++ b/mimic-iii/concepts/durations/epinephrine_durations.sql @@ -198,11 +198,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221289 -- epinephrine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/isuprel_durations.sql b/mimic-iii/concepts/durations/isuprel_durations.sql index b1191881..a468232f 100644 --- a/mimic-iii/concepts/durations/isuprel_durations.sql +++ b/mimic-iii/concepts/durations/isuprel_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 227692 -- Isuprel and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/milrinone_durations.sql b/mimic-iii/concepts/durations/milrinone_durations.sql index 69005edd..4840ba85 100644 --- a/mimic-iii/concepts/durations/milrinone_durations.sql +++ b/mimic-iii/concepts/durations/milrinone_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221986 -- milrinone and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/norepinephrine_durations.sql b/mimic-iii/concepts/durations/norepinephrine_durations.sql index 90b1d949..55317ea1 100644 --- a/mimic-iii/concepts/durations/norepinephrine_durations.sql +++ b/mimic-iii/concepts/durations/norepinephrine_durations.sql @@ -194,11 +194,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221906 -- norepinephrine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -220,6 +248,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/phenylephrine_durations.sql b/mimic-iii/concepts/durations/phenylephrine_durations.sql index 014e364d..c105d358 100644 --- a/mimic-iii/concepts/durations/phenylephrine_durations.sql +++ b/mimic-iii/concepts/durations/phenylephrine_durations.sql @@ -198,11 +198,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221749 -- phenylephrine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/vasopressin_durations.sql b/mimic-iii/concepts/durations/vasopressin_durations.sql index 8f93f19f..35b5f688 100644 --- a/mimic-iii/concepts/durations/vasopressin_durations.sql +++ b/mimic-iii/concepts/durations/vasopressin_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 222315 -- vasopressin and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/vasopressor_durations.sql b/mimic-iii/concepts/durations/vasopressor_durations.sql index 352c9ae8..64691d8a 100644 --- a/mimic-iii/concepts/durations/vasopressor_durations.sql +++ b/mimic-iii/concepts/durations/vasopressor_durations.sql @@ -263,11 +263,12 @@ ORDER BY s1.icustay_id, s1.starttime -- do not need to group by itemid because we group by linkorderid , vasomv as ( + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + -- Overlapping intervals are merged in vasomv_grp. select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + , starttime, endtime from io_mv - group by icustay_id, linkorderid ) , vasomv_grp as ( From 31c72b6364d238e70bc45db7433e19850b7d0ce0 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:01:28 +0300 Subject: [PATCH 33/49] fix: merge overlapping MetaVision arterial/central line intervals Collapse overlapping procedureevents_mv rows so duration tables no longer emit nested intervals (#371). Co-authored-by: Cursor --- .../durations/arterial_line_durations.sql | 41 ++++++++++++++++--- .../durations/central_line_durations.sql | 41 ++++++++++++++++--- 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/mimic-iii/concepts/durations/arterial_line_durations.sql b/mimic-iii/concepts/durations/arterial_line_durations.sql index 2f87ef30..db11ffa3 100644 --- a/mimic-iii/concepts/durations/arterial_line_durations.sql +++ b/mimic-iii/concepts/durations/arterial_line_durations.sql @@ -163,16 +163,45 @@ with mv as group by icustay_id, arterial_line_rownum having min(charttime) != max(charttime) ) +-- collapse overlapping MetaVision arterial line intervals (#371) +, mv_dur as +( + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + , DATETIME_DIFF(MIN(t1.endtime), s1.starttime, HOUR) AS duration_hours + FROM mv s1 + INNER JOIN mv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.arterial_line = 1 + AND t1.arterial_line = 1 + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM mv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t2.arterial_line = 1 + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE s1.arterial_line = 1 + AND NOT EXISTS + ( + SELECT 1 FROM mv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s2.arterial_line = 1 + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime +) select icustay_id -- , arterial_line_rownum , starttime, endtime, duration_hours from cv_dur UNION ALL ---TODO: collapse metavision durations if they overlap select icustay_id - -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as arterial_line_rownum - , starttime, endtime - , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -from mv -where arterial_line = 1 + , starttime, endtime, duration_hours +from mv_dur order by icustay_id, starttime; diff --git a/mimic-iii/concepts/durations/central_line_durations.sql b/mimic-iii/concepts/durations/central_line_durations.sql index 3f837c19..2dc9c11d 100644 --- a/mimic-iii/concepts/durations/central_line_durations.sql +++ b/mimic-iii/concepts/durations/central_line_durations.sql @@ -158,16 +158,45 @@ with mv as group by icustay_id, central_line_rownum having min(charttime) != max(charttime) ) +-- collapse overlapping MetaVision central line intervals (#371) +, mv_dur as +( + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + , DATETIME_DIFF(MIN(t1.endtime), s1.starttime, HOUR) AS duration_hours + FROM mv s1 + INNER JOIN mv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.central_line = 1 + AND t1.central_line = 1 + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM mv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t2.central_line = 1 + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE s1.central_line = 1 + AND NOT EXISTS + ( + SELECT 1 FROM mv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s2.central_line = 1 + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime +) select icustay_id -- , central_line_rownum , starttime, endtime, duration_hours from cv_dur UNION ALL ---TODO: collapse metavision durations if they overlap select icustay_id - -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as central_line_rownum - , starttime, endtime - , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -from mv -where central_line = 1 + , starttime, endtime, duration_hours +from mv_dur order by icustay_id, starttime; From 9e59a048d4ea477ba88364e4c17c65dca84a4041 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:14:32 +0300 Subject: [PATCH 34/49] fix: Elixhauser no_drg excludes primary and RTRIMs ICD-9 codes WHERE seq_num = 1 contradicted the header (and Quan); padded CHAR codes also failed unpadded BETWEEN ranges (#1077). Co-authored-by: Cursor --- .../concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql index c9f10a78..b3944540 100644 --- a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql +++ b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql @@ -390,8 +390,11 @@ select hadm_id, seq_num, icd9_code when icd9_code = '3091' then 1 when icd9_code = '311' then 1 end as depress /* Depression */ -from `physionet-data.mimiciii_clinical.diagnoses_icd` icd -WHERE seq_num = 1 +from ( + select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code + from `physionet-data.mimiciii_clinical.diagnoses_icd` + where seq_num != 1 +) icd ) -- collapse the icd9_code specific flags into hadm_id specific flags -- this groups comorbidities together for a single patient admission From 15c2f7251f16fbd1daef62ef9a24faa345b333f4 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:14:32 +0300 Subject: [PATCH 35/49] fix: AHRQ Elixhauser with DRG uses all diagnoses, RTRIM codes Primary-only filtering skipped secondary codes that DRG exclusions need; RTRIM matches padded ICD-9 storage (#1077). Co-authored-by: Cursor --- mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql index 603a3a4c..245f0eb8 100644 --- a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql +++ b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql @@ -384,8 +384,10 @@ select hadm_id, seq_num, icd9_code when icd9_code = '3091' then 1 when icd9_code = '311' then 1 end as depress /* Depression */ -from `physionet-data.mimiciii_clinical.diagnoses_icd` icd -WHERE seq_num = 1 +from ( + select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code + from `physionet-data.mimiciii_clinical.diagnoses_icd` +) icd ) -- collapse the icd9_code specific flags into hadm_id specific flags -- this groups comorbidities together for a single patient admission From afe7707ecb94c8350f0957157c7158c9c3a96e10 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:15:54 +0300 Subject: [PATCH 36/49] fix: RTRIM ICD-9 codes in Quan Elixhauser Padded codes break exact and prefix matches the same way as AHRQ (#1077). Co-authored-by: Cursor --- mimic-iii/concepts/comorbidity/elixhauser_quan.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mimic-iii/concepts/comorbidity/elixhauser_quan.sql b/mimic-iii/concepts/comorbidity/elixhauser_quan.sql index 18db8c00..7c5a1f2e 100644 --- a/mimic-iii/concepts/comorbidity/elixhauser_quan.sql +++ b/mimic-iii/concepts/comorbidity/elixhauser_quan.sql @@ -174,7 +174,10 @@ select hadm_id, seq_num, icd9_code when SUBSTR(icd9_code, 1, 4) in ('2962','2963','2965','3004') then 1 when SUBSTR(icd9_code, 1, 3) in ('309','311') then 1 else 0 end as depress /* Depression */ -from `physionet-data.mimiciii_clinical.diagnoses_icd` icd +from ( + select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code + from `physionet-data.mimiciii_clinical.diagnoses_icd` +) icd where seq_num != 1 -- we do not include the primary icd-9 code ) -- collapse the icd9_code specific flags into hadm_id specific flags From 70cfd84df1342abd90371546da50f2e5134f5eaf Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:24:42 +0300 Subject: [PATCH 37/49] fix: drop LODS BUN >= 7.50 mg/dL score band That cutpoint is mmol/L; on MIMIC BUN (mg/dL) it scored normal values as 1 (#1065). Co-authored-by: Cursor --- mimic-iii/concepts/severityscores/lods.sql | 3 ++- mimic-iv/concepts/score/lods.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mimic-iii/concepts/severityscores/lods.sql b/mimic-iii/concepts/severityscores/lods.sql index b5c3a7cb..e95d28c8 100644 --- a/mimic-iii/concepts/severityscores/lods.sql +++ b/mimic-iii/concepts/severityscores/lods.sql @@ -174,7 +174,8 @@ select when urineoutput >= 10000.0 then 3 when creatinine_max >= 1.20 then 1 when bun_max >= 17.0 then 1 - when bun_max >= 7.50 then 1 + -- note: do not use bun_max >= 7.50; that is a mmol/L cutpoint + -- incorrectly applied to MIMIC BUN in mg/dL (#1065) else 0 end as renal diff --git a/mimic-iv/concepts/score/lods.sql b/mimic-iv/concepts/score/lods.sql index bc105402..1bde1f88 100644 --- a/mimic-iv/concepts/score/lods.sql +++ b/mimic-iv/concepts/score/lods.sql @@ -168,7 +168,8 @@ WITH cpap AS ( WHEN urineoutput >= 10000.0 THEN 3 WHEN creatinine_max >= 1.20 THEN 1 WHEN bun_max >= 17.0 THEN 1 - WHEN bun_max >= 7.50 THEN 1 + -- note: do not use bun_max >= 7.50; that is a mmol/L cutpoint + -- incorrectly applied to MIMIC BUN in mg/dL (#1065) ELSE 0 END AS renal From 5163678441508268f7757fee8750f5420514196a Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:24:42 +0300 Subject: [PATCH 38/49] fix: drop concept tables via bq ls CSV, not cut -f3 Pretty-format column cutting mis-parses table names when rebuilding derived. Co-authored-by: Cursor --- mimic-iv/concepts/make_concepts.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/mimic-iv/concepts/make_concepts.sh b/mimic-iv/concepts/make_concepts.sh index cc633fc3..42be9d22 100644 --- a/mimic-iv/concepts/make_concepts.sh +++ b/mimic-iv/concepts/make_concepts.sh @@ -8,16 +8,15 @@ export MIMIC_VERSION="3.1" # note: max_rows=1 *displays* only one row, but all rows are inserted into the destination table BQ_OPTIONS='--quiet --headless --max_rows=0 --use_legacy_sql=False --replace' -# drop the existing tables in the target dataset -for TABLE in `bq ls physionet-data:${TARGET_DATASET} | cut -d' ' -f3`; -do - # skip the first line of dashes - if [[ "${TABLE:0:2}" == '--' ]]; then - continue - fi +# drop existing tables (CSV format: cut -f3 on pretty ls mis-parses names) +while IFS= read -r TABLE; do + [ -z "${TABLE}" ] && continue echo "Dropping table ${TARGET_DATASET}.${TABLE}" - bq rm -f -q ${TARGET_DATASET}.${TABLE} -done + bq rm -f -q "${TARGET_DATASET}.${TABLE}" +done < <( + bq ls --format=csv --max_results=10000 "physionet-data:${TARGET_DATASET}" \ + | awk -F, 'NR > 1 { print $1 }' +) # create a _version table to store the mimic-iv version, git commit hash, and latest git tag GIT_COMMIT_HASH=$(git rev-parse HEAD) From 6ce72f9cbfbab61ec879c9d80478c8a4af943f80 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:24:42 +0300 Subject: [PATCH 39/49] fix: skip echo charttime parse when time regex misses PARSE_DATETIME on a NULL-concatenated string failed rows without a time. Co-authored-by: Cursor --- mimic-iii/concepts/echo_data.sql | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/mimic-iii/concepts/echo_data.sql b/mimic-iii/concepts/echo_data.sql index 679aa9eb..1582d4ed 100644 --- a/mimic-iii/concepts/echo_data.sql +++ b/mimic-iii/concepts/echo_data.sql @@ -10,13 +10,16 @@ select ROW_ID -- however, the time is available in the echo text, e.g.: -- , substring(ne.text, 'Date/Time: [\[\]0-9*-]+ at ([0-9:]+)') as TIMESTAMP -- we can therefore impute it and re-create charttime - , PARSE_DATETIME - ( - '%Y-%m-%d%H:%M:%S', - FORMAT_DATE('%Y-%m-%d', chartdate) - || REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') - || ':00' - ) AS charttime + , CASE + WHEN REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') IS NOT NULL + THEN PARSE_DATETIME( + '%Y-%m-%d%H:%M:%S', + FORMAT_DATE('%Y-%m-%d', chartdate) + || REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') + || ':00' + ) + ELSE NULL + END AS charttime -- explanation of below substring: -- 'Indication: ' - matched verbatim From d7ec3d540aecb38f4242f8fddd3ab7902dd69be4 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:35:00 +0300 Subject: [PATCH 40/49] fix: transpile DATETIME_DIFF MONTH for Postgres MONTH previously KeyError'd in _SECONDS_PER_UNIT; emit year*12+month. Co-authored-by: Cursor --- src/mimic_utils/sqlglot_dialects/postgres.py | 16 ++++++++++++++++ tests/test_transpile.py | 4 ++++ 2 files changed, 20 insertions(+) diff --git a/src/mimic_utils/sqlglot_dialects/postgres.py b/src/mimic_utils/sqlglot_dialects/postgres.py index d3369855..509b369b 100644 --- a/src/mimic_utils/sqlglot_dialects/postgres.py +++ b/src/mimic_utils/sqlglot_dialects/postgres.py @@ -23,8 +23,11 @@ def _unit(expression: exp.Expression, default: str = "DAY") -> str: # The logic is as follows: # * DAY -> difference of the two calendar dates (date subtraction = whole days) # * YEAR -> difference of the two calendar years +# * MONTH -> year*12 + month difference (calendar month boundaries) # * sub-day units -> truncate both operands to the unit (which makes the # elapsed seconds an exact multiple of the unit) then divide. +# WEEK is intentionally unsupported: BigQuery week starts (SUNDAY/ISO) do not +# match PostgreSQL DATE_TRUNC('week') Monday semantics. # https://cloud.google.com/bigquery/docs/reference/standard-sql/datetime_functions#datetime_diff _SECONDS_PER_UNIT = {"SECOND": 1, "MINUTE": 60, "HOUR": 3600} @@ -39,6 +42,19 @@ def _datetime_diff_sql(self: Postgres.Generator, expression: exp.Expression) -> return f"(CAST({end} AS DATE) - CAST({start} AS DATE))" if unit == "YEAR": return f"CAST(EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start}) AS BIGINT)" + if unit == "MONTH": + # Calendar month boundaries (matches BigQuery DATETIME_DIFF MONTH). + return ( + "CAST((EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start})) * 12 " + "+ (EXTRACT(MONTH FROM {end}) - EXTRACT(MONTH FROM {start})) AS BIGINT)" + ).format(end=end, start=start) + if unit not in _SECONDS_PER_UNIT: + # WEEK (and WEEK(SUNDAY)/ISO) need BigQuery's week-start semantics; + # refuse rather than emit a silently wrong days/7 division. + raise ValueError( + f"Unsupported DATETIME_DIFF unit {unit!r}; " + "expected SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR" + ) lo = unit.lower() factor = _SECONDS_PER_UNIT[unit] diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 8e18bad3..151d075f 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -52,6 +52,9 @@ def t(bq: str, dialect: str) -> str: "SELECT (CAST(a.dischtime AS DATE) - CAST(a.admittime AS DATE)) FROM t AS a"), ("diff_year_pg", "SELECT DATETIME_DIFF(a.b, a.c, YEAR) FROM t a", "postgres", "SELECT CAST(EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c) AS BIGINT) FROM t AS a"), + ("diff_month_pg", "SELECT DATETIME_DIFF(a.b, a.c, MONTH) FROM t a", "postgres", + "SELECT CAST((EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c)) * 12 " + "+ (EXTRACT(MONTH FROM a.b) - EXTRACT(MONTH FROM a.c)) AS BIGINT) FROM t AS a"), # DuckDB handles DATETIME_DIFF natively (boundary count), operands swapped ("diff_hour_duckdb", "SELECT DATETIME_DIFF(a.outtime, a.intime, HOUR) FROM t a", "duckdb", "SELECT DATE_DIFF('HOUR', a.intime, a.outtime) FROM t AS a"), @@ -219,6 +222,7 @@ def test_mimic_iii_concept_transpiles_and_reparses(sql_file, dialect): ("2150-01-02 01:00:00", "2150-01-01 23:00:00", "DAY", 1), # crosses midnight ("2150-01-01 23:00:00", "2150-01-01 01:00:00", "DAY", 0), # same day ("2155-06-15 00:00:00", "2150-01-01 00:00:00", "YEAR", 5), + ("2150-04-01 00:00:00", "2150-01-15 00:00:00", "MONTH", 3), ("2150-01-01 00:01:00", "2150-01-01 00:00:59", "MINUTE", 1), ("2150-01-01 00:00:30", "2150-01-01 00:00:10", "SECOND", 20), ("2150-01-01 01:00:00", "2150-01-01 03:00:00", "HOUR", -2), # negative direction From 1fe4366bbea75a1d6c52fbe2552c03e1a09d0c71 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:35:00 +0300 Subject: [PATCH 41/49] fix: include CCO/Arctic Sun temps in III vitals Same itemids as IV vitalsign (#1358): 226329, 227632, 227634. Co-authored-by: Cursor --- mimic-iii/concepts/firstday/vitals_first_day.sql | 7 +++++-- mimic-iii/concepts/pivot/pivoted_vital.sql | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/mimic-iii/concepts/firstday/vitals_first_day.sql b/mimic-iii/concepts/firstday/vitals_first_day.sql index a77def4d..659278e5 100644 --- a/mimic-iii/concepts/firstday/vitals_first_day.sql +++ b/mimic-iii/concepts/firstday/vitals_first_day.sql @@ -38,7 +38,7 @@ FROM ( when itemid in (456,52,6702,443,220052,220181,225312) and valuenum > 0 and valuenum < 300 then 4 -- MeanBP when itemid in (615,618,220210,224690) and valuenum > 0 and valuenum < 70 then 5 -- RespRate when itemid in (223761,678) and valuenum > 70 and valuenum < 120 then 6 -- TempF, converted to degC in valuenum call - when itemid in (223762,676) and valuenum > 10 and valuenum < 50 then 6 -- TempC + when itemid in (223762,676,226329,227632,227634) and valuenum > 10 and valuenum < 50 then 6 -- TempC when itemid in (646,220277) and valuenum > 0 and valuenum <= 100 then 7 -- SpO2 when itemid in (807,811,1529,3745,3744,225664,220621,226537) and valuenum > 0 then 8 -- Glucose @@ -110,7 +110,10 @@ FROM ( 223762, -- "Temperature Celsius" 676, -- "Temperature C" 223761, -- "Temperature Fahrenheit" - 678 -- "Temperature F" + 678, -- "Temperature F" + 226329, -- Blood Temperature CCO (C) + 227632, -- Arctic Sun/Alsius Temp # 1 C + 227634 -- Arctic Sun/Alsius Temp # 2 C ) ) pvt diff --git a/mimic-iii/concepts/pivot/pivoted_vital.sql b/mimic-iii/concepts/pivot/pivoted_vital.sql index 27efa0db..12dd7f51 100644 --- a/mimic-iii/concepts/pivot/pivoted_vital.sql +++ b/mimic-iii/concepts/pivot/pivoted_vital.sql @@ -11,7 +11,7 @@ with ce as , (case when itemid in (456,52,6702,443,220052,220181,225312) and valuenum > 0 and valuenum < 300 then valuenum else null end) as meanbp , (case when itemid in (615,618,220210,224690) and valuenum > 0 and valuenum < 70 then valuenum else null end) as resprate , (case when itemid in (223761,678) and valuenum > 70 and valuenum < 120 then (valuenum-32)/1.8 -- converted to degC in valuenum call - when itemid in (223762,676) and valuenum > 10 and valuenum < 50 then valuenum else null end) as tempc + when itemid in (223762,676,226329,227632,227634) and valuenum > 10 and valuenum < 50 then valuenum else null end) as tempc , (case when itemid in (646,220277) and valuenum > 0 and valuenum <= 100 then valuenum else null end) as spo2 , (case when itemid in (807,811,1529,3745,3744,225664,220621,226537) and valuenum > 0 then valuenum else null end) as glucose FROM `physionet-data.mimiciii_clinical.chartevents` ce @@ -74,7 +74,10 @@ with ce as 223762, -- "Temperature Celsius" 676, -- "Temperature C" 223761, -- "Temperature Fahrenheit" - 678 -- "Temperature F" + 678, -- "Temperature F" + 226329, -- Blood Temperature CCO (C) + 227632, -- Arctic Sun/Alsius Temp # 1 C + 227634 -- Arctic Sun/Alsius Temp # 2 C ) ) From 0d80d2b5880a0eb2fdb3e43fa94c409b1a49cc3c Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:35:01 +0300 Subject: [PATCH 42/49] fix: include CCO/Arctic Sun temps in aline_vitals Co-authored-by: Cursor --- mimic-iii/notebooks/aline/aline_vitals.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mimic-iii/notebooks/aline/aline_vitals.sql b/mimic-iii/notebooks/aline/aline_vitals.sql index 9ee60b79..033be2c6 100644 --- a/mimic-iii/notebooks/aline/aline_vitals.sql +++ b/mimic-iii/notebooks/aline/aline_vitals.sql @@ -10,13 +10,13 @@ with vitals_stg0 as , case -- MAP, Temperature, HR, CVP, SpO2, when itemid in (456,52,6702,443,220052,220181,225312) then 'MAP' - when itemid in (223762,676,223761,678) then 'Temperature' + when itemid in (223762,676,223761,678,226329,227632,227634) then 'Temperature' when itemid in (211,220045) then 'HeartRate' when itemid in (646,220277) then 'SpO2' else null end as label , case when itemid in (223761,678) and ((valuenum-32)/1.8)<10 then null - when itemid in (223762,676) and valuenum < 10 then null + when itemid in (223762,676,226329,227632,227634) and valuenum < 10 then null -- convert F to C when itemid in (223761,678) then (valuenum-32)/1.8 -- sanity checks on data - one outliter with spo2 < 25 @@ -32,6 +32,7 @@ with vitals_stg0 as ( 456,52,6702,443,220052,220181,225312 -- map , 223762,676,223761,678 -- temp + , 226329,227632,227634 -- CCO / Arctic Sun temp (C) , 211,220045 -- hr , 646,220277 -- spo2 ) From fc005949d211e120861aa0755982ee31c39eb666 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:46:08 +0300 Subject: [PATCH 43/49] fix: skip echo height PARSE when time regex misses Co-authored-by: Cursor --- mimic-iii/concepts/pivot/pivoted_height.sql | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mimic-iii/concepts/pivot/pivoted_height.sql b/mimic-iii/concepts/pivot/pivoted_height.sql index 61452739..b481b0aa 100644 --- a/mimic-iii/concepts/pivot/pivoted_height.sql +++ b/mimic-iii/concepts/pivot/pivoted_height.sql @@ -83,12 +83,16 @@ WITH ht_in AS subject_id -- extract the time of the note from the text itself -- add this to the structured date in the chartdate column - , PARSE_DATETIME('%b-%d-%Y%H:%M', - CONCAT( - FORMAT_DATE("%b-%d-%Y", chartdate), - REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') - ) - ) AS charttime + , CASE + WHEN REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') IS NOT NULL + THEN PARSE_DATETIME('%b-%d-%Y%H:%M', + CONCAT( + FORMAT_DATE("%b-%d-%Y", chartdate), + REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') + ) + ) + ELSE NULL + END AS charttime -- sometimes numeric values contain de-id numbers, e.g. [** Numeric Identifier **] -- this case is used to ignore that text , case From b836c5a242781d9af305de464636fcdbeea109ad Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:46:08 +0300 Subject: [PATCH 44/49] fix: include CCO/Arctic Sun temps in aline-aws vitals Co-authored-by: Cursor --- mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql b/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql index 6d30869c..7aadd4da 100644 --- a/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql +++ b/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql @@ -9,13 +9,13 @@ with vitals_stg0 as , case -- MAP, Temperature, HR, CVP, SpO2, when itemid in (456,52,6702,443,220052,220181,225312) then 'MAP' - when itemid in (223762,676,223761,678) then 'Temperature' + when itemid in (223762,676,223761,678,226329,227632,227634) then 'Temperature' when itemid in (211,220045) then 'HeartRate' when itemid in (646,220277) then 'SpO2' else null end as label , case when itemid in (223761,678) and ((valuenum-32)/1.8)<10 then null - when itemid in (223762,676) and valuenum < 10 then null + when itemid in (223762,676,226329,227632,227634) and valuenum < 10 then null -- convert F to C when itemid in (223761,678) then (valuenum-32)/1.8 -- sanity checks on data - one outliter with spo2 < 25 @@ -31,6 +31,7 @@ with vitals_stg0 as ( 456,52,6702,443,220052,220181,225312 -- map , 223762,676,223761,678 -- temp + , 226329,227632,227634 -- CCO / Arctic Sun temp (C) , 211,220045 -- hr , 646,220277 -- spo2 ) From 1cab27c6446ccbfb3f28e9ef0e455294c7dbcdb9 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:46:08 +0300 Subject: [PATCH 45/49] fix: convert SAPS glucose/BUN thresholds to mg/dL SAPS 1984 publishes mmol/L cutpoints; MIMIC labs are mg/dL. Apply PhysioNet Challenge 2012 conversions (glucose x18, BUN x2.8). Co-authored-by: Cursor --- mimic-iii/concepts/severityscores/saps.sql | 33 ++++++++++++---------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/mimic-iii/concepts/severityscores/saps.sql b/mimic-iii/concepts/severityscores/saps.sql index ec961f5d..f4d01eba 100644 --- a/mimic-iii/concepts/severityscores/saps.sql +++ b/mimic-iii/concepts/severityscores/saps.sql @@ -195,13 +195,15 @@ select , case when bun_max is null then null - when bun_max >= 55.0 then 4 - when bun_max >= 36.0 then 3 - when bun_max >= 29.0 then 2 - when bun_max >= 7.50 then 1 - when bun_min < 3.5 then 1 - when bun_max >= 3.5 and bun_max < 7.5 - and bun_min >= 3.5 and bun_min < 7.5 + -- SAPS 1984 publishes urea in mmol/L; MIMIC BUN is mg/dL (×2.8). + -- Same conversion as PhysioNet Challenge 2012 saps_score.m. + when bun_max >= 154.0 then 4 + when bun_max >= 100.8 then 3 + when bun_max >= 81.2 then 2 + when bun_max >= 21.0 then 1 + when bun_min < 9.8 then 1 + when bun_max >= 9.8 and bun_max < 21.0 + and bun_min >= 9.8 and bun_min < 21.0 then 0 end as bun_score @@ -231,14 +233,15 @@ select , case when glucose_max is null then null - when glucose_max >= 44.5 then 4 - when glucose_min < 1.6 then 4 - when glucose_max >= 27.8 then 3 - when glucose_min < 2.8 then 3 - when glucose_min < 3.9 then 2 - when glucose_max >= 14.0 then 1 - when glucose_max >= 3.9 and glucose_max < 14.0 - and glucose_min >= 3.9 and glucose_min < 14.0 + -- SAPS 1984 glucose is mmol/L; MIMIC glucose is mg/dL (×18). + when glucose_max >= 801.0 then 4 + when glucose_min < 28.8 then 4 + when glucose_max >= 500.4 then 3 + when glucose_min < 50.4 then 3 + when glucose_min < 70.2 then 2 + when glucose_max >= 252.0 then 1 + when glucose_max >= 70.2 and glucose_max < 252.0 + and glucose_min >= 70.2 and glucose_min < 252.0 then 0 end as glucose_score From cb5f9ac811679ece99dca4b25df6e3c3f42fea28 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:55:15 +0300 Subject: [PATCH 46/49] fix: build icustay_times/hours before pivoted_oasis pivoted_oasis reads mimiciii_derived.icustay_hours; make-concepts never created it (or its icustay_times dependency). Co-authored-by: Cursor --- mimic-iii/concepts/make-concepts.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mimic-iii/concepts/make-concepts.sh b/mimic-iii/concepts/make-concepts.sh index 01e15f00..7e74e3b8 100644 --- a/mimic-iii/concepts/make-concepts.sh +++ b/mimic-iii/concepts/make-concepts.sh @@ -22,12 +22,17 @@ echo 'Running queries in 10 directories.' echo 'Directory 1: demographics' bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.heightweight" < "demographics/heightweight.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.icustay_detail" < "demographics/icustay_detail.sql" +# icustay_hours (and pivoted_oasis) need icustay_times first +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.icustay_times" < "demographics/icustay_times.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.icustay_hours" < "demographics/icustay_hours.sql" # Durations (usually of treatments) echo 'Directory 2: durations' bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_classification" < "durations/ventilation_classification.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_durations" < "durations/ventilation_durations.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.crrt_durations" < "durations/crrt_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.arterial_line_durations" < "durations/arterial_line_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.central_line_durations" < "durations/central_line_durations.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.adenosine_durations" < "durations/adenosine_durations.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dobutamine_durations" < "durations/dobutamine_durations.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dopamine_durations" < "durations/dopamine_durations.sql" @@ -46,6 +51,7 @@ bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.epinephrine_dose" < bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.norepinephrine_dose" < "durations/norepinephrine_dose.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.phenylephrine_dose" < "durations/phenylephrine_dose.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vasopressin_dose" < "durations/vasopressin_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.neuroblock_dose" < "durations/neuroblock_dose.sql" # "pivoted" tables which have icustay_id / timestamp as the primary key echo 'Directory 3: pivoted tables' From ca766fc8aa9c8d6de8c2af062cbed62bb7357e24 Mon Sep 17 00:00:00 2001 From: Taksh Date: Fri, 24 Jul 2026 09:25:58 +0300 Subject: [PATCH 47/49] fix: include pivoted_oasis in postgres/duckdb concept builds SQL existed but make scripts never ran it (#830). Co-authored-by: Cursor --- mimic-iii/concepts_duckdb/duckdb.sql | 2 ++ mimic-iii/concepts_postgres/postgres-make-concepts.sql | 1 + 2 files changed, 3 insertions(+) diff --git a/mimic-iii/concepts_duckdb/duckdb.sql b/mimic-iii/concepts_duckdb/duckdb.sql index dc49fd86..2635b2ad 100644 --- a/mimic-iii/concepts_duckdb/duckdb.sql +++ b/mimic-iii/concepts_duckdb/duckdb.sql @@ -93,6 +93,8 @@ .read pivot/pivoted_vital.sql .print 'pivot/pivoted_bg_art.sql' .read pivot/pivoted_bg_art.sql +.print 'pivot/pivoted_oasis.sql' +.read pivot/pivoted_oasis.sql .print 'pivot/pivoted_sofa.sql' .read pivot/pivoted_sofa.sql diff --git a/mimic-iii/concepts_postgres/postgres-make-concepts.sql b/mimic-iii/concepts_postgres/postgres-make-concepts.sql index 3779bef6..318b3da1 100644 --- a/mimic-iii/concepts_postgres/postgres-make-concepts.sql +++ b/mimic-iii/concepts_postgres/postgres-make-concepts.sql @@ -61,6 +61,7 @@ SET search_path TO mimiciii_derived, mimiciii; \i pivot/pivoted_uo.sql \i pivot/pivoted_vital.sql \i pivot/pivoted_bg_art.sql +\i pivot/pivoted_oasis.sql \i pivot/pivoted_sofa.sql -- comorbidity From 00ff2df2d6a7254426d5928afd6523f461c9c872 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sat, 25 Jul 2026 08:55:25 +0300 Subject: [PATCH 48/49] chore: drop concept SQL changes for separate PRs Per maintainer feedback on #2124, keep load-path / transpile / CI fixes here and move concept logic to individual PRs. Co-authored-by: Cursor --- mimic-iii/concepts/README.md | 11 +++-- .../comorbidity/elixhauser_ahrq_v37.sql | 6 +-- .../elixhauser_ahrq_v37_no_drg.sql | 7 +--- .../concepts/comorbidity/elixhauser_quan.sql | 5 +-- mimic-iii/concepts/diagnosis/README.md | 4 +- mimic-iii/concepts/diagnosis/ccs_dx.sql | 2 +- .../durations/adenosine_durations.sql | 34 ++------------- .../durations/arterial_line_durations.sql | 41 +++---------------- .../durations/central_line_durations.sql | 41 +++---------------- .../durations/dobutamine_durations.sql | 34 ++------------- .../concepts/durations/dopamine_durations.sql | 34 ++------------- .../durations/epinephrine_durations.sql | 34 ++------------- .../concepts/durations/isuprel_durations.sql | 34 ++------------- .../durations/milrinone_durations.sql | 34 ++------------- .../concepts/durations/neuroblock_dose.sql | 10 ++--- .../durations/norepinephrine_durations.sql | 34 ++------------- .../durations/phenylephrine_durations.sql | 34 ++------------- .../durations/vasopressin_durations.sql | 34 ++------------- .../durations/vasopressor_durations.sql | 5 +-- mimic-iii/concepts/echo_data.sql | 17 ++++---- .../concepts/firstday/vitals_first_day.sql | 7 +--- mimic-iii/concepts/pivot/pivoted_height.sql | 16 +++----- mimic-iii/concepts/pivot/pivoted_sofa.sql | 2 +- mimic-iii/concepts/pivot/pivoted_vital.sql | 7 +--- mimic-iii/concepts/severityscores/README.md | 31 +++++++------- mimic-iii/concepts/severityscores/lods.sql | 3 +- mimic-iii/concepts/severityscores/saps.sql | 33 +++++++-------- mimic-iii/concepts/severityscores/sofa.sql | 2 +- mimic-iii/concepts_duckdb/README.md | 2 +- .../diagnosis/ccs_multi_dx.sql | 7 ++-- .../durations/arterial_line_durations.sql | 2 +- .../durations/central_line_durations.sql | 2 +- .../organfailure/kdigo_stages.sql | 5 ++- mimic-iii/concepts_postgres/README.md | 2 +- .../concepts_postgres/diagnosis/ccs_dx.sql | 2 +- .../diagnosis/ccs_multi_dx.sql | 7 ++-- .../durations/arterial_line_durations.sql | 2 +- .../durations/central_line_durations.sql | 2 +- .../organfailure/kdigo_stages.sql | 13 +++--- mimic-iii/notebooks/aline-aws/README.md | 2 +- .../notebooks/aline-aws/aline-awsathena.ipynb | 2 +- .../aline-aws/aline_labs-awsathena.sql | 4 +- .../aline-aws/aline_vitals-awsathena.sql | 7 ++-- mimic-iii/notebooks/aline/README.md | 2 +- mimic-iii/notebooks/aline/aline.ipynb | 2 +- mimic-iii/notebooks/aline/aline_labs.sql | 4 +- mimic-iii/notebooks/aline/aline_vitals.sql | 7 ++-- mimic-iv/concepts/README.md | 4 +- mimic-iv/concepts/comorbidity/charlson.sql | 16 +++----- mimic-iv/concepts/firstday/first_day_sofa.sql | 4 +- .../concepts/measurement/oxygen_delivery.sql | 5 +-- .../measurement/urine_output_rate.sql | 25 ++++------- mimic-iv/concepts/measurement/vitalsign.sql | 6 +-- mimic-iv/concepts/score/lods.sql | 3 +- mimic-iv/concepts/score/sofa.sql | 4 +- .../demographics/icustay_hourly.sql | 2 +- 56 files changed, 175 insertions(+), 526 deletions(-) diff --git a/mimic-iii/concepts/README.md b/mimic-iii/concepts/README.md index b9059325..039474bc 100644 --- a/mimic-iii/concepts/README.md +++ b/mimic-iii/concepts/README.md @@ -5,7 +5,7 @@ The scripts are intended to be run against the BigQuery instantiation of MIMIC-I Concepts are categorized into folders if possible, otherwise they remain in the top-level directory. A table of contents is provided below: [List of concepts](#list-of-concepts). -You can read about cloud access to MIMIC-III, including via Google BigQuery, on the [cloud page](https://mimic.physionet.org/gettingstarted/cloud/). +You can read about cloud access to MIMIC-III, including via Google BigQuery, on the [cloud page](https://mimic.mit.edu/docs/gettingstarted/cloud/). The rest of this README describes: @@ -14,7 +14,7 @@ The rest of this README describes: ## Generating the concepts in BigQuery -You do not need to generate the concepts if you are using BigQuery! They have already been generated for you. If you have access to MIMIC-III on BigQuery, look under `physionet-data.mimic_derived`. If you would like to generate the concepts again, for example on your own dataset, you must modify the `TARGET_DATASET` variable within the [make-concepts.sh](/concepts/make-concepts.sh) script. The script assumes you have installed and configured the [Google Cloud SDK](https://cloud.google.com/sdk/docs/install). +You do not need to generate the concepts if you are using BigQuery! They have already been generated for you. If you have access to MIMIC-III on BigQuery, look under `physionet-data.mimiciii_derived`. If you would like to generate the concepts again, for example on your own dataset, you must modify the `TARGET_DATASET` variable within the [make-concepts.sh](/mimic-iii/concepts/make-concepts.sh) script. The script assumes you have installed and configured the [Google Cloud SDK](https://cloud.google.com/sdk/docs/install). ## Generating the concepts in PostgreSQL or DuckDB @@ -42,6 +42,7 @@ Folder | Table | Description --- | --- | --- . | [echo_data](echo_data.sql) | Text extracted from echocardiography reports using regular expressions. . | [code_status](code_status.sql) | Whether the patient has restrictions on life saving resuscitation. +. | [rrt](rrt.sql) | Whether the patient received any dialysis during their ICU stay. comorbidity | [elixhauser_ahrq_v37](comorbidity/elixhauser_ahrq_v37.sql) | Comorbidities in categories proposed by Elixhauser et al. AHRQ produced the mapping. comorbidity | [elixhauser_ahrq_v37_no_drg](comorbidity/elixhauser_ahrq_v37_no_drg.sql) | As above, but DRG codes are not used to exclude primary conditions. comorbidity | [elixhauser_quan](comorbidity/elixhauser_quan.sql) | Comorbidities in categories proposed by Elixhauser et al. using an algorithm by Quan et al. @@ -76,7 +77,7 @@ durations | [neuroblock_dose](durations/neuroblock_dose.sql) durations | [norepinephrine_dose](durations/norepinephrine_dose.sql) | Dose administered with start/stop times for norepinephrine. durations | [phenylephrine_dose](durations/phenylephrine_dose.sql) | Dose administered with start/stop times for phenylephrine. durations | [vasopressin_dose](durations/vasopressin_dose.sql) | Dose administered with start/stop times for vasopressin. -durations | [ventilation_classification](durations/ventilation_classification.sql) | Classifies patient settings as implying mechanical ventilation. +durations | [ventilation_classification](durations/ventilation_classification.sql) | Classifies ventilator settings (tidal volume, PEEP, etc.) as implying mechanical ventilation. durations | [ventilation_durations](durations/ventilation_durations.sql) | Start and stop times for mechanical ventilation. durations | [weight_durations](durations/weight_durations.sql) | Start and stop times for daily weight measurements. **firstday** | | The first day subfolder contains scripts to summarizes a patient's health on their first ICU day. @@ -93,6 +94,8 @@ firstday | [weight_first_day](firstday/weight_first_day.sql) **fluid_balance** | | Tables which track fluid input and output for the patient. fluid_balance | [colloid_bolus](fluid_balance/colloid_bolus.sql) | Times at which a patient received a bolus of colloidal fluid. fluid_balance | [crystalloid_bolus](fluid_balance/crystalloid_bolus.sql) | Times at which a patient received a bolus of crystalloid fluid. +fluid_balance | [ffp_transfusion](fluid_balance/ffp_transfusion.sql) | Times at which a patient received a transfusion of fresh frozen plasma. +fluid_balance | [rbc_transfusion](fluid_balance/rbc_transfusion.sql) | Times at which a patient received a transfusion of red blood cells. fluid_balance | [urine_output](fluid_balance/urine_output.sql) | Urine output for a patient with the time of documentation. **organfailure** | | Summarizations of the degree of organ failure for single organ systems. organfailure | [kdigo_creatinine](organfailure/kdigo_creatinine.sql) | Creatinine values with baseline creatinine as defined by KDIGO. @@ -103,6 +106,7 @@ organfailure | [kdigo_stages_7day](organfailure/kdigo_stages_7day.sql) organfailure | [meld](organfailure/meld.sql) | The MELD score, often used to assess health of liver transplant candidates. **pivot** | | Pivoted views contain the patient `icustay_id`, the `charttime`, and a number of variables. They are useful to acquiring a time series of values for patient stays. pivot | [pivoted_bg](pivot/pivoted_bg.sql) | Blood gas measurements. +pivot | [pivoted_bg_art](pivot/pivoted_bg_art.sql) | Arterial blood gas measurements, explicitly labelled or inferred from SpO2. pivot | [pivoted_fio2](pivot/pivoted_fio2.sql) | Fraction of inspired oxygen. pivot | [pivoted_gcs](pivot/pivoted_gcs.sql) | Glasgow Coma Scale. pivot | [pivoted_height](pivot/pivoted_height.sql) | Height. @@ -113,7 +117,6 @@ pivot | [pivoted_oasis](pivot/pivoted_oasis.sql) pivot | [pivoted_rrt](pivot/pivoted_rrt.sql) | Renal replacement therapy. pivot | [pivoted_sofa](pivot/pivoted_sofa.sql) | The Sequential Organ Failure Assessment (SOFA) scale. pivot | [pivoted_uo](pivot/pivoted_uo.sql) | Urine output. -pivot | [pivoted_vent_setting](pivot/pivoted_vent_setting.sql) | Ventilator settings (tidal volume, PEEP, etc). pivot | [pivoted_vital](pivot/pivoted_vital.sql) | Vital signs. **sepsis** | | Definitions of sepsis, a common cause of mortality for intensive care unit patients. sepsis | [angus](sepsis/angus.sql) | Sepsis defined using billing codes validated by Angus et al. diff --git a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql index 245f0eb8..603a3a4c 100644 --- a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql +++ b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql @@ -384,10 +384,8 @@ select hadm_id, seq_num, icd9_code when icd9_code = '3091' then 1 when icd9_code = '311' then 1 end as depress /* Depression */ -from ( - select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code - from `physionet-data.mimiciii_clinical.diagnoses_icd` -) icd +from `physionet-data.mimiciii_clinical.diagnoses_icd` icd +WHERE seq_num = 1 ) -- collapse the icd9_code specific flags into hadm_id specific flags -- this groups comorbidities together for a single patient admission diff --git a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql index b3944540..c9f10a78 100644 --- a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql +++ b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql @@ -390,11 +390,8 @@ select hadm_id, seq_num, icd9_code when icd9_code = '3091' then 1 when icd9_code = '311' then 1 end as depress /* Depression */ -from ( - select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code - from `physionet-data.mimiciii_clinical.diagnoses_icd` - where seq_num != 1 -) icd +from `physionet-data.mimiciii_clinical.diagnoses_icd` icd +WHERE seq_num = 1 ) -- collapse the icd9_code specific flags into hadm_id specific flags -- this groups comorbidities together for a single patient admission diff --git a/mimic-iii/concepts/comorbidity/elixhauser_quan.sql b/mimic-iii/concepts/comorbidity/elixhauser_quan.sql index 7c5a1f2e..18db8c00 100644 --- a/mimic-iii/concepts/comorbidity/elixhauser_quan.sql +++ b/mimic-iii/concepts/comorbidity/elixhauser_quan.sql @@ -174,10 +174,7 @@ select hadm_id, seq_num, icd9_code when SUBSTR(icd9_code, 1, 4) in ('2962','2963','2965','3004') then 1 when SUBSTR(icd9_code, 1, 3) in ('309','311') then 1 else 0 end as depress /* Depression */ -from ( - select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code - from `physionet-data.mimiciii_clinical.diagnoses_icd` -) icd +from `physionet-data.mimiciii_clinical.diagnoses_icd` icd where seq_num != 1 -- we do not include the primary icd-9 code ) -- collapse the icd9_code specific flags into hadm_id specific flags diff --git a/mimic-iii/concepts/diagnosis/README.md b/mimic-iii/concepts/diagnosis/README.md index ae6d0230..b323d76e 100644 --- a/mimic-iii/concepts/diagnosis/README.md +++ b/mimic-iii/concepts/diagnosis/README.md @@ -4,7 +4,7 @@ The Clinical Classification Software (CCS) categorizes ICD-9 coded diagnoses int The `ccs_multi_dx.csv.gz` data file must be uploaded to `physionet-data.mimiciii_derived.ccs_multi_dx`. -The BigQuery schema definition is available in this folder as [ccs_multi_dx.json](/ccs_multi_dx.json). +The BigQuery schema definition is available in this folder as [ccs_multi_dx.json](ccs_multi_dx.json). ## Creation of the ccs_multi_dx.csv.gz file @@ -24,7 +24,7 @@ Use Python to convert all apostrophes in `ccs_multi_dx_tool_2015.csv` into doubl ```python import pandas as pd -df = pd.read_csv('ccs_multi_dx_tool_2015.csv.gz') +df = pd.read_csv('ccs_multi_dx_tool_2015.csv') # remove apostrophes from header names and relabel df.rename(columns={"'ICD-9-CM CODE'": "icd9_code", "'CCS LVL 1'": "ccs_level1", "'CCS LVL 1 LABEL'": "ccs_group1", "'CCS LVL 2'": "ccs_level2", "'CCS LVL 2 LABEL'": "ccs_group2", "'CCS LVL 3'": "ccs_level3", "'CCS LVL 3 LABEL'": "ccs_group3", "'CCS LVL 4'": "ccs_level4", "'CCS LVL 4 LABEL'": "ccs_group4", }, inplace=True) diff --git a/mimic-iii/concepts/diagnosis/ccs_dx.sql b/mimic-iii/concepts/diagnosis/ccs_dx.sql index d67062d1..24781e52 100644 --- a/mimic-iii/concepts/diagnosis/ccs_dx.sql +++ b/mimic-iii/concepts/diagnosis/ccs_dx.sql @@ -1,5 +1,5 @@ -- add in matched ID, name, and ccs_id --- matched id (ccs_mid): the ccs ID with the hierachy, e.g. 7.1.2.1 +-- matched id (ccs_mid): the ccs ID with the hierarchy, e.g. 7.1.2.1 -- name (ccs_name): the most granular CCS category the diagnosis is in -- ID (ccs_id): the CCS identifier for the ICD-9 code (integer) SELECT diff --git a/mimic-iii/concepts/durations/adenosine_durations.sql b/mimic-iii/concepts/durations/adenosine_durations.sql index 8859198a..4bd2afcd 100644 --- a/mimic-iii/concepts/durations/adenosine_durations.sql +++ b/mimic-iii/concepts/durations/adenosine_durations.sql @@ -198,39 +198,11 @@ and ( select icustay_id, linkorderid - -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). - , starttime, endtime + , min(starttime) as starttime, max(endtime) as endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221282 -- adenosine and statusdescription != 'Rewritten' -- only valid orders -) - -, vasomv_grp as -( - -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). - SELECT - s1.icustay_id - , s1.starttime - , MIN(t1.endtime) AS endtime - FROM vasomv s1 - INNER JOIN vasomv t1 - ON s1.icustay_id = t1.icustay_id - AND s1.starttime <= t1.endtime - AND NOT EXISTS - ( - SELECT 1 FROM vasomv t2 - WHERE t1.icustay_id = t2.icustay_id - AND t1.endtime >= t2.starttime - AND t1.endtime < t2.endtime - ) - WHERE NOT EXISTS - ( - SELECT 1 FROM vasomv s2 - WHERE s1.icustay_id = s2.icustay_id - AND s1.starttime > s2.starttime - AND s1.starttime <= s2.endtime - ) - GROUP BY s1.icustay_id, s1.starttime + group by icustay_id, linkorderid ) select @@ -252,6 +224,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv_grp + vasomv order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/arterial_line_durations.sql b/mimic-iii/concepts/durations/arterial_line_durations.sql index db11ffa3..2f87ef30 100644 --- a/mimic-iii/concepts/durations/arterial_line_durations.sql +++ b/mimic-iii/concepts/durations/arterial_line_durations.sql @@ -163,45 +163,16 @@ with mv as group by icustay_id, arterial_line_rownum having min(charttime) != max(charttime) ) --- collapse overlapping MetaVision arterial line intervals (#371) -, mv_dur as -( - SELECT - s1.icustay_id - , s1.starttime - , MIN(t1.endtime) AS endtime - , DATETIME_DIFF(MIN(t1.endtime), s1.starttime, HOUR) AS duration_hours - FROM mv s1 - INNER JOIN mv t1 - ON s1.icustay_id = t1.icustay_id - AND s1.arterial_line = 1 - AND t1.arterial_line = 1 - AND s1.starttime <= t1.endtime - AND NOT EXISTS - ( - SELECT 1 FROM mv t2 - WHERE t1.icustay_id = t2.icustay_id - AND t2.arterial_line = 1 - AND t1.endtime >= t2.starttime - AND t1.endtime < t2.endtime - ) - WHERE s1.arterial_line = 1 - AND NOT EXISTS - ( - SELECT 1 FROM mv s2 - WHERE s1.icustay_id = s2.icustay_id - AND s2.arterial_line = 1 - AND s1.starttime > s2.starttime - AND s1.starttime <= s2.endtime - ) - GROUP BY s1.icustay_id, s1.starttime -) select icustay_id -- , arterial_line_rownum , starttime, endtime, duration_hours from cv_dur UNION ALL +--TODO: collapse metavision durations if they overlap select icustay_id - , starttime, endtime, duration_hours -from mv_dur + -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as arterial_line_rownum + , starttime, endtime + , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours +from mv +where arterial_line = 1 order by icustay_id, starttime; diff --git a/mimic-iii/concepts/durations/central_line_durations.sql b/mimic-iii/concepts/durations/central_line_durations.sql index 2dc9c11d..3f837c19 100644 --- a/mimic-iii/concepts/durations/central_line_durations.sql +++ b/mimic-iii/concepts/durations/central_line_durations.sql @@ -158,45 +158,16 @@ with mv as group by icustay_id, central_line_rownum having min(charttime) != max(charttime) ) --- collapse overlapping MetaVision central line intervals (#371) -, mv_dur as -( - SELECT - s1.icustay_id - , s1.starttime - , MIN(t1.endtime) AS endtime - , DATETIME_DIFF(MIN(t1.endtime), s1.starttime, HOUR) AS duration_hours - FROM mv s1 - INNER JOIN mv t1 - ON s1.icustay_id = t1.icustay_id - AND s1.central_line = 1 - AND t1.central_line = 1 - AND s1.starttime <= t1.endtime - AND NOT EXISTS - ( - SELECT 1 FROM mv t2 - WHERE t1.icustay_id = t2.icustay_id - AND t2.central_line = 1 - AND t1.endtime >= t2.starttime - AND t1.endtime < t2.endtime - ) - WHERE s1.central_line = 1 - AND NOT EXISTS - ( - SELECT 1 FROM mv s2 - WHERE s1.icustay_id = s2.icustay_id - AND s2.central_line = 1 - AND s1.starttime > s2.starttime - AND s1.starttime <= s2.endtime - ) - GROUP BY s1.icustay_id, s1.starttime -) select icustay_id -- , central_line_rownum , starttime, endtime, duration_hours from cv_dur UNION ALL +--TODO: collapse metavision durations if they overlap select icustay_id - , starttime, endtime, duration_hours -from mv_dur + -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as central_line_rownum + , starttime, endtime + , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours +from mv +where central_line = 1 order by icustay_id, starttime; diff --git a/mimic-iii/concepts/durations/dobutamine_durations.sql b/mimic-iii/concepts/durations/dobutamine_durations.sql index 7063348c..bdac0706 100644 --- a/mimic-iii/concepts/durations/dobutamine_durations.sql +++ b/mimic-iii/concepts/durations/dobutamine_durations.sql @@ -195,39 +195,11 @@ and ( select icustay_id, linkorderid - -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). - , starttime, endtime + , min(starttime) as starttime, max(endtime) as endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221653 -- dobutamine and statusdescription != 'Rewritten' -- only valid orders -) - -, vasomv_grp as -( - -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). - SELECT - s1.icustay_id - , s1.starttime - , MIN(t1.endtime) AS endtime - FROM vasomv s1 - INNER JOIN vasomv t1 - ON s1.icustay_id = t1.icustay_id - AND s1.starttime <= t1.endtime - AND NOT EXISTS - ( - SELECT 1 FROM vasomv t2 - WHERE t1.icustay_id = t2.icustay_id - AND t1.endtime >= t2.starttime - AND t1.endtime < t2.endtime - ) - WHERE NOT EXISTS - ( - SELECT 1 FROM vasomv s2 - WHERE s1.icustay_id = s2.icustay_id - AND s1.starttime > s2.starttime - AND s1.starttime <= s2.endtime - ) - GROUP BY s1.icustay_id, s1.starttime + group by icustay_id, linkorderid ) select @@ -249,6 +221,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv_grp + vasomv order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/dopamine_durations.sql b/mimic-iii/concepts/durations/dopamine_durations.sql index 1f5672ce..65ccea38 100644 --- a/mimic-iii/concepts/durations/dopamine_durations.sql +++ b/mimic-iii/concepts/durations/dopamine_durations.sql @@ -198,39 +198,11 @@ and ( select icustay_id, linkorderid - -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). - , starttime, endtime + , min(starttime) as starttime, max(endtime) as endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221662 -- dopamine and statusdescription != 'Rewritten' -- only valid orders -) - -, vasomv_grp as -( - -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). - SELECT - s1.icustay_id - , s1.starttime - , MIN(t1.endtime) AS endtime - FROM vasomv s1 - INNER JOIN vasomv t1 - ON s1.icustay_id = t1.icustay_id - AND s1.starttime <= t1.endtime - AND NOT EXISTS - ( - SELECT 1 FROM vasomv t2 - WHERE t1.icustay_id = t2.icustay_id - AND t1.endtime >= t2.starttime - AND t1.endtime < t2.endtime - ) - WHERE NOT EXISTS - ( - SELECT 1 FROM vasomv s2 - WHERE s1.icustay_id = s2.icustay_id - AND s1.starttime > s2.starttime - AND s1.starttime <= s2.endtime - ) - GROUP BY s1.icustay_id, s1.starttime + group by icustay_id, linkorderid ) select @@ -252,6 +224,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv_grp + vasomv order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/epinephrine_durations.sql b/mimic-iii/concepts/durations/epinephrine_durations.sql index 43cfae00..a51a4d55 100644 --- a/mimic-iii/concepts/durations/epinephrine_durations.sql +++ b/mimic-iii/concepts/durations/epinephrine_durations.sql @@ -198,39 +198,11 @@ and ( select icustay_id, linkorderid - -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). - , starttime, endtime + , min(starttime) as starttime, max(endtime) as endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221289 -- epinephrine and statusdescription != 'Rewritten' -- only valid orders -) - -, vasomv_grp as -( - -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). - SELECT - s1.icustay_id - , s1.starttime - , MIN(t1.endtime) AS endtime - FROM vasomv s1 - INNER JOIN vasomv t1 - ON s1.icustay_id = t1.icustay_id - AND s1.starttime <= t1.endtime - AND NOT EXISTS - ( - SELECT 1 FROM vasomv t2 - WHERE t1.icustay_id = t2.icustay_id - AND t1.endtime >= t2.starttime - AND t1.endtime < t2.endtime - ) - WHERE NOT EXISTS - ( - SELECT 1 FROM vasomv s2 - WHERE s1.icustay_id = s2.icustay_id - AND s1.starttime > s2.starttime - AND s1.starttime <= s2.endtime - ) - GROUP BY s1.icustay_id, s1.starttime + group by icustay_id, linkorderid ) select @@ -252,6 +224,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv_grp + vasomv order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/isuprel_durations.sql b/mimic-iii/concepts/durations/isuprel_durations.sql index a468232f..b1191881 100644 --- a/mimic-iii/concepts/durations/isuprel_durations.sql +++ b/mimic-iii/concepts/durations/isuprel_durations.sql @@ -195,39 +195,11 @@ and ( select icustay_id, linkorderid - -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). - , starttime, endtime + , min(starttime) as starttime, max(endtime) as endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 227692 -- Isuprel and statusdescription != 'Rewritten' -- only valid orders -) - -, vasomv_grp as -( - -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). - SELECT - s1.icustay_id - , s1.starttime - , MIN(t1.endtime) AS endtime - FROM vasomv s1 - INNER JOIN vasomv t1 - ON s1.icustay_id = t1.icustay_id - AND s1.starttime <= t1.endtime - AND NOT EXISTS - ( - SELECT 1 FROM vasomv t2 - WHERE t1.icustay_id = t2.icustay_id - AND t1.endtime >= t2.starttime - AND t1.endtime < t2.endtime - ) - WHERE NOT EXISTS - ( - SELECT 1 FROM vasomv s2 - WHERE s1.icustay_id = s2.icustay_id - AND s1.starttime > s2.starttime - AND s1.starttime <= s2.endtime - ) - GROUP BY s1.icustay_id, s1.starttime + group by icustay_id, linkorderid ) select @@ -249,6 +221,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv_grp + vasomv order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/milrinone_durations.sql b/mimic-iii/concepts/durations/milrinone_durations.sql index 4840ba85..69005edd 100644 --- a/mimic-iii/concepts/durations/milrinone_durations.sql +++ b/mimic-iii/concepts/durations/milrinone_durations.sql @@ -195,39 +195,11 @@ and ( select icustay_id, linkorderid - -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). - , starttime, endtime + , min(starttime) as starttime, max(endtime) as endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221986 -- milrinone and statusdescription != 'Rewritten' -- only valid orders -) - -, vasomv_grp as -( - -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). - SELECT - s1.icustay_id - , s1.starttime - , MIN(t1.endtime) AS endtime - FROM vasomv s1 - INNER JOIN vasomv t1 - ON s1.icustay_id = t1.icustay_id - AND s1.starttime <= t1.endtime - AND NOT EXISTS - ( - SELECT 1 FROM vasomv t2 - WHERE t1.icustay_id = t2.icustay_id - AND t1.endtime >= t2.starttime - AND t1.endtime < t2.endtime - ) - WHERE NOT EXISTS - ( - SELECT 1 FROM vasomv s2 - WHERE s1.icustay_id = s2.icustay_id - AND s1.starttime > s2.starttime - AND s1.starttime <= s2.endtime - ) - GROUP BY s1.icustay_id, s1.starttime + group by icustay_id, linkorderid ) select @@ -249,6 +221,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv_grp + vasomv order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/neuroblock_dose.sql b/mimic-iii/concepts/durations/neuroblock_dose.sql index a1ce5c9a..098b557e 100644 --- a/mimic-iii/concepts/durations/neuroblock_dose.sql +++ b/mimic-iii/concepts/durations/neuroblock_dose.sql @@ -28,7 +28,7 @@ with drugmv as , 1 as drug -- the 'stopped' column indicates if a drug has been disconnected - , max(case when stopped in ('Stopped', CONCAT('D/C', CHR(39), 'd')) then 1 else 0 end) as drug_stopped + , max(case when stopped in ('Stopped','D/C''d') then 1 else 0 end) as drug_stopped -- we only include continuous infusions, therefore expect a rate , max(case @@ -68,7 +68,7 @@ with drugmv as , 1 as drug -- the 'stopped' column indicates if a drug has been disconnected - , max(case when stopped in ('Stopped', CONCAT('D/C', CHR(39), 'd')) then 1 else 0 end) as drug_stopped + , max(case when stopped in ('Stopped','D/C''d') then 1 else 0 end) as drug_stopped , max(case when valuenum <= 10 then 0 else 1 end) as drug_null -- educated guess! @@ -179,11 +179,7 @@ select ) = 1 then 1 - when DATETIME_DIFF( - CHARTTIME, - LAG(CHARTTIME, 1) OVER (partition by icustay_id, drug order by charttime), - HOUR - ) > 8 then 1 + when (CHARTTIME - (LAG(CHARTTIME, 1) OVER (partition by icustay_id, drug order by charttime))) > (interval '8 hours') then 1 else null end as drug_start diff --git a/mimic-iii/concepts/durations/norepinephrine_durations.sql b/mimic-iii/concepts/durations/norepinephrine_durations.sql index 55317ea1..90b1d949 100644 --- a/mimic-iii/concepts/durations/norepinephrine_durations.sql +++ b/mimic-iii/concepts/durations/norepinephrine_durations.sql @@ -194,39 +194,11 @@ and ( select icustay_id, linkorderid - -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). - , starttime, endtime + , min(starttime) as starttime, max(endtime) as endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221906 -- norepinephrine and statusdescription != 'Rewritten' -- only valid orders -) - -, vasomv_grp as -( - -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). - SELECT - s1.icustay_id - , s1.starttime - , MIN(t1.endtime) AS endtime - FROM vasomv s1 - INNER JOIN vasomv t1 - ON s1.icustay_id = t1.icustay_id - AND s1.starttime <= t1.endtime - AND NOT EXISTS - ( - SELECT 1 FROM vasomv t2 - WHERE t1.icustay_id = t2.icustay_id - AND t1.endtime >= t2.starttime - AND t1.endtime < t2.endtime - ) - WHERE NOT EXISTS - ( - SELECT 1 FROM vasomv s2 - WHERE s1.icustay_id = s2.icustay_id - AND s1.starttime > s2.starttime - AND s1.starttime <= s2.endtime - ) - GROUP BY s1.icustay_id, s1.starttime + group by icustay_id, linkorderid ) select @@ -248,6 +220,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv_grp + vasomv order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/phenylephrine_durations.sql b/mimic-iii/concepts/durations/phenylephrine_durations.sql index c105d358..014e364d 100644 --- a/mimic-iii/concepts/durations/phenylephrine_durations.sql +++ b/mimic-iii/concepts/durations/phenylephrine_durations.sql @@ -198,39 +198,11 @@ and ( select icustay_id, linkorderid - -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). - , starttime, endtime + , min(starttime) as starttime, max(endtime) as endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221749 -- phenylephrine and statusdescription != 'Rewritten' -- only valid orders -) - -, vasomv_grp as -( - -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). - SELECT - s1.icustay_id - , s1.starttime - , MIN(t1.endtime) AS endtime - FROM vasomv s1 - INNER JOIN vasomv t1 - ON s1.icustay_id = t1.icustay_id - AND s1.starttime <= t1.endtime - AND NOT EXISTS - ( - SELECT 1 FROM vasomv t2 - WHERE t1.icustay_id = t2.icustay_id - AND t1.endtime >= t2.starttime - AND t1.endtime < t2.endtime - ) - WHERE NOT EXISTS - ( - SELECT 1 FROM vasomv s2 - WHERE s1.icustay_id = s2.icustay_id - AND s1.starttime > s2.starttime - AND s1.starttime <= s2.endtime - ) - GROUP BY s1.icustay_id, s1.starttime + group by icustay_id, linkorderid ) select @@ -252,6 +224,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv_grp + vasomv order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/vasopressin_durations.sql b/mimic-iii/concepts/durations/vasopressin_durations.sql index 35b5f688..8f93f19f 100644 --- a/mimic-iii/concepts/durations/vasopressin_durations.sql +++ b/mimic-iii/concepts/durations/vasopressin_durations.sql @@ -195,39 +195,11 @@ and ( select icustay_id, linkorderid - -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). - , starttime, endtime + , min(starttime) as starttime, max(endtime) as endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 222315 -- vasopressin and statusdescription != 'Rewritten' -- only valid orders -) - -, vasomv_grp as -( - -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). - SELECT - s1.icustay_id - , s1.starttime - , MIN(t1.endtime) AS endtime - FROM vasomv s1 - INNER JOIN vasomv t1 - ON s1.icustay_id = t1.icustay_id - AND s1.starttime <= t1.endtime - AND NOT EXISTS - ( - SELECT 1 FROM vasomv t2 - WHERE t1.icustay_id = t2.icustay_id - AND t1.endtime >= t2.starttime - AND t1.endtime < t2.endtime - ) - WHERE NOT EXISTS - ( - SELECT 1 FROM vasomv s2 - WHERE s1.icustay_id = s2.icustay_id - AND s1.starttime > s2.starttime - AND s1.starttime <= s2.endtime - ) - GROUP BY s1.icustay_id, s1.starttime + group by icustay_id, linkorderid ) select @@ -249,6 +221,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv_grp + vasomv order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/vasopressor_durations.sql b/mimic-iii/concepts/durations/vasopressor_durations.sql index 64691d8a..352c9ae8 100644 --- a/mimic-iii/concepts/durations/vasopressor_durations.sql +++ b/mimic-iii/concepts/durations/vasopressor_durations.sql @@ -263,12 +263,11 @@ ORDER BY s1.icustay_id, s1.starttime -- do not need to group by itemid because we group by linkorderid , vasomv as ( - -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). - -- Overlapping intervals are merged in vasomv_grp. select icustay_id, linkorderid - , starttime, endtime + , min(starttime) as starttime, max(endtime) as endtime from io_mv + group by icustay_id, linkorderid ) , vasomv_grp as ( diff --git a/mimic-iii/concepts/echo_data.sql b/mimic-iii/concepts/echo_data.sql index 1582d4ed..679aa9eb 100644 --- a/mimic-iii/concepts/echo_data.sql +++ b/mimic-iii/concepts/echo_data.sql @@ -10,16 +10,13 @@ select ROW_ID -- however, the time is available in the echo text, e.g.: -- , substring(ne.text, 'Date/Time: [\[\]0-9*-]+ at ([0-9:]+)') as TIMESTAMP -- we can therefore impute it and re-create charttime - , CASE - WHEN REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') IS NOT NULL - THEN PARSE_DATETIME( - '%Y-%m-%d%H:%M:%S', - FORMAT_DATE('%Y-%m-%d', chartdate) - || REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') - || ':00' - ) - ELSE NULL - END AS charttime + , PARSE_DATETIME + ( + '%Y-%m-%d%H:%M:%S', + FORMAT_DATE('%Y-%m-%d', chartdate) + || REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') + || ':00' + ) AS charttime -- explanation of below substring: -- 'Indication: ' - matched verbatim diff --git a/mimic-iii/concepts/firstday/vitals_first_day.sql b/mimic-iii/concepts/firstday/vitals_first_day.sql index 659278e5..a77def4d 100644 --- a/mimic-iii/concepts/firstday/vitals_first_day.sql +++ b/mimic-iii/concepts/firstday/vitals_first_day.sql @@ -38,7 +38,7 @@ FROM ( when itemid in (456,52,6702,443,220052,220181,225312) and valuenum > 0 and valuenum < 300 then 4 -- MeanBP when itemid in (615,618,220210,224690) and valuenum > 0 and valuenum < 70 then 5 -- RespRate when itemid in (223761,678) and valuenum > 70 and valuenum < 120 then 6 -- TempF, converted to degC in valuenum call - when itemid in (223762,676,226329,227632,227634) and valuenum > 10 and valuenum < 50 then 6 -- TempC + when itemid in (223762,676) and valuenum > 10 and valuenum < 50 then 6 -- TempC when itemid in (646,220277) and valuenum > 0 and valuenum <= 100 then 7 -- SpO2 when itemid in (807,811,1529,3745,3744,225664,220621,226537) and valuenum > 0 then 8 -- Glucose @@ -110,10 +110,7 @@ FROM ( 223762, -- "Temperature Celsius" 676, -- "Temperature C" 223761, -- "Temperature Fahrenheit" - 678, -- "Temperature F" - 226329, -- Blood Temperature CCO (C) - 227632, -- Arctic Sun/Alsius Temp # 1 C - 227634 -- Arctic Sun/Alsius Temp # 2 C + 678 -- "Temperature F" ) ) pvt diff --git a/mimic-iii/concepts/pivot/pivoted_height.sql b/mimic-iii/concepts/pivot/pivoted_height.sql index b481b0aa..61452739 100644 --- a/mimic-iii/concepts/pivot/pivoted_height.sql +++ b/mimic-iii/concepts/pivot/pivoted_height.sql @@ -83,16 +83,12 @@ WITH ht_in AS subject_id -- extract the time of the note from the text itself -- add this to the structured date in the chartdate column - , CASE - WHEN REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') IS NOT NULL - THEN PARSE_DATETIME('%b-%d-%Y%H:%M', - CONCAT( - FORMAT_DATE("%b-%d-%Y", chartdate), - REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') - ) - ) - ELSE NULL - END AS charttime + , PARSE_DATETIME('%b-%d-%Y%H:%M', + CONCAT( + FORMAT_DATE("%b-%d-%Y", chartdate), + REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') + ) + ) AS charttime -- sometimes numeric values contain de-id numbers, e.g. [** Numeric Identifier **] -- this case is used to ignore that text , case diff --git a/mimic-iii/concepts/pivot/pivoted_sofa.sql b/mimic-iii/concepts/pivot/pivoted_sofa.sql index c7947368..6d4940fd 100644 --- a/mimic-iii/concepts/pivot/pivoted_sofa.sql +++ b/mimic-iii/concepts/pivot/pivoted_sofa.sql @@ -228,7 +228,7 @@ WITH co AS -- Cardiovascular , cast(case when rate_dopamine > 15 or rate_epinephrine > 0.1 or rate_norepinephrine > 0.1 then 4 - when rate_dopamine > 5 or (rate_epinephrine > 0 and rate_epinephrine <= 0.1) or (rate_norepinephrine > 0 and rate_norepinephrine <= 0.1) then 3 + when rate_dopamine > 5 or rate_epinephrine <= 0.1 or rate_norepinephrine <= 0.1 then 3 when rate_dopamine > 0 or rate_dobutamine > 0 then 2 when meanbp_min < 70 then 1 when coalesce(meanbp_min, rate_dopamine, rate_dobutamine, rate_epinephrine, rate_norepinephrine) is null then null diff --git a/mimic-iii/concepts/pivot/pivoted_vital.sql b/mimic-iii/concepts/pivot/pivoted_vital.sql index 12dd7f51..27efa0db 100644 --- a/mimic-iii/concepts/pivot/pivoted_vital.sql +++ b/mimic-iii/concepts/pivot/pivoted_vital.sql @@ -11,7 +11,7 @@ with ce as , (case when itemid in (456,52,6702,443,220052,220181,225312) and valuenum > 0 and valuenum < 300 then valuenum else null end) as meanbp , (case when itemid in (615,618,220210,224690) and valuenum > 0 and valuenum < 70 then valuenum else null end) as resprate , (case when itemid in (223761,678) and valuenum > 70 and valuenum < 120 then (valuenum-32)/1.8 -- converted to degC in valuenum call - when itemid in (223762,676,226329,227632,227634) and valuenum > 10 and valuenum < 50 then valuenum else null end) as tempc + when itemid in (223762,676) and valuenum > 10 and valuenum < 50 then valuenum else null end) as tempc , (case when itemid in (646,220277) and valuenum > 0 and valuenum <= 100 then valuenum else null end) as spo2 , (case when itemid in (807,811,1529,3745,3744,225664,220621,226537) and valuenum > 0 then valuenum else null end) as glucose FROM `physionet-data.mimiciii_clinical.chartevents` ce @@ -74,10 +74,7 @@ with ce as 223762, -- "Temperature Celsius" 676, -- "Temperature C" 223761, -- "Temperature Fahrenheit" - 678, -- "Temperature F" - 226329, -- Blood Temperature CCO (C) - 227632, -- Arctic Sun/Alsius Temp # 1 C - 227634 -- Arctic Sun/Alsius Temp # 2 C + 678 -- "Temperature F" ) ) diff --git a/mimic-iii/concepts/severityscores/README.md b/mimic-iii/concepts/severityscores/README.md index 9ede5159..e407cbbc 100644 --- a/mimic-iii/concepts/severityscores/README.md +++ b/mimic-iii/concepts/severityscores/README.md @@ -3,19 +3,24 @@ This folder generates materialized views for a variety of severity of illness scores (detailed below). The queries make use of materialized views which aggregate data from the first day of a patient's ICU stay. To run the code, it is necessary to: -1. Run the scripts which generate the constituent materialized views - * echodata - generated by /concepts/echo-data.sql - * ventdurations - generated by /concepts/durations/ventilation-durations.sql - (needed for subsequent view) - * vitalsfirstday - generated by /concepts/firstday/vitals-first-day.sql - * uofirstday - generated by /concepts/firstday/urine-output-first-day.sql - * ventfirstday - generated by /concepts/firstday/ventilated-first-day.sql - * gcsfirstday - generated by /concepts/firstday/gcs-first-day.sql - * labsfirstday - generated by /concepts/firstday/labs-first-day.sql - * bloodgasfirstday - generated by /concepts/firstday/blood-gas-first-day.sql - (needed for subsequent view) - * bloodgasfirstdayarterial - generated by /concepts/firstday/blood-gas-first-day-arterial.sql +1. Run the scripts which generate the constituent tables + * echo_data - generated by [/mimic-iii/concepts/echo_data.sql](/mimic-iii/concepts/echo_data.sql) + * ventilation_durations - generated by [/mimic-iii/concepts/durations/ventilation_durations.sql](/mimic-iii/concepts/durations/ventilation_durations.sql) - (needed for subsequent table) + * vitals_first_day - generated by [/mimic-iii/concepts/firstday/vitals_first_day.sql](/mimic-iii/concepts/firstday/vitals_first_day.sql) + * urine_output_first_day - generated by [/mimic-iii/concepts/firstday/urine_output_first_day.sql](/mimic-iii/concepts/firstday/urine_output_first_day.sql) + * ventilation_first_day - generated by [/mimic-iii/concepts/firstday/ventilation_first_day.sql](/mimic-iii/concepts/firstday/ventilation_first_day.sql) + * gcs_first_day - generated by [/mimic-iii/concepts/firstday/gcs_first_day.sql](/mimic-iii/concepts/firstday/gcs_first_day.sql) + * labs_first_day - generated by [/mimic-iii/concepts/firstday/labs_first_day.sql](/mimic-iii/concepts/firstday/labs_first_day.sql) + * blood_gas_first_day - generated by [/mimic-iii/concepts/firstday/blood_gas_first_day.sql](/mimic-iii/concepts/firstday/blood_gas_first_day.sql) - (needed for subsequent table) + * blood_gas_first_day_arterial - generated by [/mimic-iii/concepts/firstday/blood_gas_first_day_arterial.sql](/mimic-iii/concepts/firstday/blood_gas_first_day_arterial.sql) 2. Run the script for the severity of illness score you are interested in + * APS III - apsiii.sql + * LODS - lods.sql (and a modified version, mlods.sql) * OASIS - oasis.sql + * qSOFA - qsofa.sql * SAPS - saps.sql + * SAPS II - sapsii.sql + * SIRS - sirs.sql * SOFA - sofa.sql The scripts were written and tested in PostgreSQL 9.4.4. The scripts are written to follow the SQL standard as close as possible, and there should be minimal changes necessary for them to operate under different languages. Key changes likely involve: @@ -48,11 +53,7 @@ Note that the organ-specific morbidity scores are highlighted as a useful compli The following severity scores have yet to be implemented, but are desirable, and contributions are welcome. -## SAPS II - -The Simplified Acute Physiology Score II (SAPS II) published in 1993 (Le Gall, 1993), aimed to rectify two issues with SAPS. First, the variable selection process in SAPS was done by clinical judgement, whereas SAPS II utilised univariate feature selection to filter out features uncorrelated with hospital mortality. Second, there was no model for calculating a probability of mortality from SAPS. SAPS II was published with calibration coefficients which allowed conversion of the integer score into a risk of mortality which ranged between zero and one. - -### SAPS III +## SAPS III The Simplified Acute Physiology Score III (SAPS III) is the latest model published and attempts to account for poor calibration of SAPS II found in later studies (Metnitz, 2005). While SAPS II was developed on ICUs in western Europe, SAPS III included ICUs worldwide. diff --git a/mimic-iii/concepts/severityscores/lods.sql b/mimic-iii/concepts/severityscores/lods.sql index e95d28c8..b5c3a7cb 100644 --- a/mimic-iii/concepts/severityscores/lods.sql +++ b/mimic-iii/concepts/severityscores/lods.sql @@ -174,8 +174,7 @@ select when urineoutput >= 10000.0 then 3 when creatinine_max >= 1.20 then 1 when bun_max >= 17.0 then 1 - -- note: do not use bun_max >= 7.50; that is a mmol/L cutpoint - -- incorrectly applied to MIMIC BUN in mg/dL (#1065) + when bun_max >= 7.50 then 1 else 0 end as renal diff --git a/mimic-iii/concepts/severityscores/saps.sql b/mimic-iii/concepts/severityscores/saps.sql index f4d01eba..ec961f5d 100644 --- a/mimic-iii/concepts/severityscores/saps.sql +++ b/mimic-iii/concepts/severityscores/saps.sql @@ -195,15 +195,13 @@ select , case when bun_max is null then null - -- SAPS 1984 publishes urea in mmol/L; MIMIC BUN is mg/dL (×2.8). - -- Same conversion as PhysioNet Challenge 2012 saps_score.m. - when bun_max >= 154.0 then 4 - when bun_max >= 100.8 then 3 - when bun_max >= 81.2 then 2 - when bun_max >= 21.0 then 1 - when bun_min < 9.8 then 1 - when bun_max >= 9.8 and bun_max < 21.0 - and bun_min >= 9.8 and bun_min < 21.0 + when bun_max >= 55.0 then 4 + when bun_max >= 36.0 then 3 + when bun_max >= 29.0 then 2 + when bun_max >= 7.50 then 1 + when bun_min < 3.5 then 1 + when bun_max >= 3.5 and bun_max < 7.5 + and bun_min >= 3.5 and bun_min < 7.5 then 0 end as bun_score @@ -233,15 +231,14 @@ select , case when glucose_max is null then null - -- SAPS 1984 glucose is mmol/L; MIMIC glucose is mg/dL (×18). - when glucose_max >= 801.0 then 4 - when glucose_min < 28.8 then 4 - when glucose_max >= 500.4 then 3 - when glucose_min < 50.4 then 3 - when glucose_min < 70.2 then 2 - when glucose_max >= 252.0 then 1 - when glucose_max >= 70.2 and glucose_max < 252.0 - and glucose_min >= 70.2 and glucose_min < 252.0 + when glucose_max >= 44.5 then 4 + when glucose_min < 1.6 then 4 + when glucose_max >= 27.8 then 3 + when glucose_min < 2.8 then 3 + when glucose_min < 3.9 then 2 + when glucose_max >= 14.0 then 1 + when glucose_max >= 3.9 and glucose_max < 14.0 + and glucose_min >= 3.9 and glucose_min < 14.0 then 0 end as glucose_score diff --git a/mimic-iii/concepts/severityscores/sofa.sql b/mimic-iii/concepts/severityscores/sofa.sql index 52cb8d5f..9afc5be1 100644 --- a/mimic-iii/concepts/severityscores/sofa.sql +++ b/mimic-iii/concepts/severityscores/sofa.sql @@ -218,7 +218,7 @@ left join `physionet-data.mimiciii_derived.gcs_first_day` gcs -- Cardiovascular , case when rate_dopamine > 15 or rate_epinephrine > 0.1 or rate_norepinephrine > 0.1 then 4 - when rate_dopamine > 5 or (rate_epinephrine > 0 and rate_epinephrine <= 0.1) or (rate_norepinephrine > 0 and rate_norepinephrine <= 0.1) then 3 + when rate_dopamine > 5 or rate_epinephrine <= 0.1 or rate_norepinephrine <= 0.1 then 3 when rate_dopamine > 0 or rate_dobutamine > 0 then 2 when meanbp_min < 70 then 1 when coalesce(meanbp_min, rate_dopamine, rate_dobutamine, rate_epinephrine, rate_norepinephrine) is null then null diff --git a/mimic-iii/concepts_duckdb/README.md b/mimic-iii/concepts_duckdb/README.md index eb23446e..f6649b9f 100644 --- a/mimic-iii/concepts_duckdb/README.md +++ b/mimic-iii/concepts_duckdb/README.md @@ -4,7 +4,7 @@ This folder contains scripts to generate useful abstractions of raw MIMIC-III da Almost all of these scripts were generated automatically from the BigQuery SQL dialect using the [sqlglot](https://github.com/tobymao/sqlglot) package. If you would like to contribute a correction, do not make it here. Instead, make your correction in the [concepts folder](/mimic-iii/concepts/) using the BigQuery SQL syntax, and regenerate this folder as described in that folder's [README](/mimic-iii/concepts/README.md). -One file is hand-written for DuckDB and has no BigQuery source: [diagnosis/ccs_multi_dx.sql](diagnosis/ccs_multi_dx.sql) loads the ICD-9 to CCS mapping from [diagnosis/ccs_multi_dx.csv.gz](diagnosis/ccs_multi_dx.csv.gz). +One file is hand-written for DuckDB and has no BigQuery source: [diagnosis/ccs_multi_dx.sql](diagnosis/ccs_multi_dx.sql) loads the ICD-9 to CCS mapping from [ccs_multi_dx.csv.gz in the concepts folder](/mimic-iii/concepts/diagnosis/ccs_multi_dx.csv.gz). ## Using these concepts diff --git a/mimic-iii/concepts_duckdb/diagnosis/ccs_multi_dx.sql b/mimic-iii/concepts_duckdb/diagnosis/ccs_multi_dx.sql index a67d5b81..89174eb3 100644 --- a/mimic-iii/concepts_duckdb/diagnosis/ccs_multi_dx.sql +++ b/mimic-iii/concepts_duckdb/diagnosis/ccs_multi_dx.sql @@ -1,13 +1,14 @@ -- THIS SCRIPT IS HAND-WRITTEN FOR DUCKDB (there is no BigQuery source). --- Loads the CCS multi-level diagnosis mapping from the CSV distributed --- alongside this script. On BigQuery the same table is loaded with: +-- Loads the CCS multi-level diagnosis mapping from the CSV stored in the +-- concepts folder (../concepts/diagnosis/ccs_multi_dx.csv.gz). On BigQuery +-- the same table is loaded with: -- bq load mimiciii_derived.ccs_multi_dx diagnosis/ccs_multi_dx.csv.gz diagnosis/ccs_multi_dx.json -- Run from the concepts_duckdb directory so the relative path resolves. DROP TABLE IF EXISTS mimiciii_derived.ccs_multi_dx; CREATE TABLE mimiciii_derived.ccs_multi_dx AS SELECT * FROM read_csv( - 'diagnosis/ccs_multi_dx.csv.gz', + '../concepts/diagnosis/ccs_multi_dx.csv.gz', header = true, columns = { 'icd9_code': 'VARCHAR(5)', diff --git a/mimic-iii/concepts_duckdb/durations/arterial_line_durations.sql b/mimic-iii/concepts_duckdb/durations/arterial_line_durations.sql index c699ee72..5958efc7 100644 --- a/mimic-iii/concepts_duckdb/durations/arterial_line_durations.sql +++ b/mimic-iii/concepts_duckdb/durations/arterial_line_durations.sql @@ -102,7 +102,7 @@ WITH mv AS ( icustay_id, charttime, charttime_lag, - charttime - charttime_lag AS arterial_line_duration, + DATE_DIFF('HOUR', charttime_lag, charttime) AS arterial_line_duration, CASE WHEN DATE_DIFF('HOUR', charttime_lag, charttime) > 16 THEN 1 ELSE 0 END AS arterial_line_new FROM cv0 ), cv2 AS ( diff --git a/mimic-iii/concepts_duckdb/durations/central_line_durations.sql b/mimic-iii/concepts_duckdb/durations/central_line_durations.sql index d31940e0..33da694a 100644 --- a/mimic-iii/concepts_duckdb/durations/central_line_durations.sql +++ b/mimic-iii/concepts_duckdb/durations/central_line_durations.sql @@ -200,7 +200,7 @@ WITH mv AS ( icustay_id, charttime, charttime_lag, - charttime - charttime_lag AS central_line_duration, + DATE_DIFF('HOUR', charttime_lag, charttime) AS central_line_duration, CASE WHEN DATE_DIFF('HOUR', charttime_lag, charttime) > 16 THEN 1 ELSE 0 END AS central_line_new FROM cv0 ), cv2 AS ( diff --git a/mimic-iii/concepts_duckdb/organfailure/kdigo_stages.sql b/mimic-iii/concepts_duckdb/organfailure/kdigo_stages.sql index 8db78065..83cdd7cb 100644 --- a/mimic-iii/concepts_duckdb/organfailure/kdigo_stages.sql +++ b/mimic-iii/concepts_duckdb/organfailure/kdigo_stages.sql @@ -12,7 +12,10 @@ WITH cr_stg AS ( THEN 3 WHEN cr.creat >= 4 AND ( - cr.creat_low_past_48hr <= 3.7 OR cr.creat >= ( + cr.creat >= ( + cr.creat_low_past_48hr + 0.3 + ) + OR cr.creat >= ( 1.5 * cr.creat_low_past_7day ) ) diff --git a/mimic-iii/concepts_postgres/README.md b/mimic-iii/concepts_postgres/README.md index 250bf012..0510e718 100644 --- a/mimic-iii/concepts_postgres/README.md +++ b/mimic-iii/concepts_postgres/README.md @@ -6,7 +6,7 @@ If you would like to contribute a correction, do not make it here. Instead, make Two files are hand-written for PostgreSQL and have no BigQuery source: -* [diagnosis/ccs_multi_dx.sql](diagnosis/ccs_multi_dx.sql) loads the ICD-9 to CCS mapping from [diagnosis/ccs_multi_dx.csv.gz](diagnosis/ccs_multi_dx.csv.gz). +* [diagnosis/ccs_multi_dx.sql](diagnosis/ccs_multi_dx.sql) loads the ICD-9 to CCS mapping from [ccs_multi_dx.csv.gz in the concepts folder](/mimic-iii/concepts/diagnosis/ccs_multi_dx.csv.gz). * [demographics/note_counts.sql](demographics/note_counts.sql) is an optional PostgreSQL-only concept which summarizes note counts per hospital admission (it is not run by the make script). ## Using these concepts diff --git a/mimic-iii/concepts_postgres/diagnosis/ccs_dx.sql b/mimic-iii/concepts_postgres/diagnosis/ccs_dx.sql index 84215817..19c352b0 100644 --- a/mimic-iii/concepts_postgres/diagnosis/ccs_dx.sql +++ b/mimic-iii/concepts_postgres/diagnosis/ccs_dx.sql @@ -1,6 +1,6 @@ -- THIS SCRIPT IS AUTOMATICALLY GENERATED. DO NOT EDIT IT DIRECTLY. DROP TABLE IF EXISTS mimiciii_derived.ccs_dx; CREATE TABLE mimiciii_derived.ccs_dx AS -/* add in matched ID, name, and ccs_id */ /* matched id (ccs_mid): the ccs ID with the hierachy, e.g. 7.1.2.1 */ /* name (ccs_name): the most granular CCS category the diagnosis is in */ /* ID (ccs_id): the CCS identifier for the ICD-9 code (integer) */ +/* add in matched ID, name, and ccs_id */ /* matched id (ccs_mid): the ccs ID with the hierarchy, e.g. 7.1.2.1 */ /* name (ccs_name): the most granular CCS category the diagnosis is in */ /* ID (ccs_id): the CCS identifier for the ICD-9 code (integer) */ SELECT icd9_code, COALESCE(ccs_level4, ccs_level3, ccs_level2, ccs_level1) AS ccs_matched_id, /* remove the trailing ccs_id from name column, i.e. "Burns [240.]" -> "Burns" */ diff --git a/mimic-iii/concepts_postgres/diagnosis/ccs_multi_dx.sql b/mimic-iii/concepts_postgres/diagnosis/ccs_multi_dx.sql index dd30e9c4..22a09d48 100644 --- a/mimic-iii/concepts_postgres/diagnosis/ccs_multi_dx.sql +++ b/mimic-iii/concepts_postgres/diagnosis/ccs_multi_dx.sql @@ -1,6 +1,7 @@ -- There is no BigQuery source for this script. --- Loads the CCS multi-level diagnosis mapping from the CSV distributed --- alongside this script. On BigQuery the same table is loaded with: +-- Loads the CCS multi-level diagnosis mapping from the CSV stored in the +-- concepts folder (../concepts/diagnosis/ccs_multi_dx.csv.gz). On BigQuery +-- the same table is loaded with: -- bq load mimiciii_derived.ccs_multi_dx diagnosis/ccs_multi_dx.csv.gz diagnosis/ccs_multi_dx.json -- Run from the concepts_postgres directory so the relative path resolves. DROP TABLE IF EXISTS mimiciii_derived.ccs_multi_dx; @@ -17,4 +18,4 @@ CREATE TABLE mimiciii_derived.ccs_multi_dx ccs_level4 VARCHAR(10), ccs_group4 VARCHAR(100) ); -\COPY mimiciii_derived.ccs_multi_dx FROM PROGRAM 'gzip -dc diagnosis/ccs_multi_dx.csv.gz' CSV HEADER; +\COPY mimiciii_derived.ccs_multi_dx FROM PROGRAM 'gzip -dc ../concepts/diagnosis/ccs_multi_dx.csv.gz' CSV HEADER; diff --git a/mimic-iii/concepts_postgres/durations/arterial_line_durations.sql b/mimic-iii/concepts_postgres/durations/arterial_line_durations.sql index 992809e3..cc40408b 100644 --- a/mimic-iii/concepts_postgres/durations/arterial_line_durations.sql +++ b/mimic-iii/concepts_postgres/durations/arterial_line_durations.sql @@ -111,7 +111,7 @@ WITH mv AS ( icustay_id, charttime, charttime_lag, /* if the current observation indicates a line is present */ /* calculate the time since the last charted line */ - charttime - charttime_lag AS arterial_line_duration, /* now we determine if the current line is "new" */ /* new == no documentation for 16 hours */ + CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', charttime) - DATE_TRUNC('hour', charttime_lag)) / 3600 AS BIGINT) AS arterial_line_duration, /* now we determine if the current line is "new" */ /* new == no documentation for 16 hours */ CASE WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', charttime) - DATE_TRUNC('hour', charttime_lag)) / 3600 AS BIGINT) > 16 THEN 1 diff --git a/mimic-iii/concepts_postgres/durations/central_line_durations.sql b/mimic-iii/concepts_postgres/durations/central_line_durations.sql index b1b6ef2c..905e17a0 100644 --- a/mimic-iii/concepts_postgres/durations/central_line_durations.sql +++ b/mimic-iii/concepts_postgres/durations/central_line_durations.sql @@ -201,7 +201,7 @@ WITH mv AS ( icustay_id, charttime, charttime_lag, /* if the current observation indicates a line is present */ /* calculate the time since the last charted line */ - charttime - charttime_lag AS central_line_duration, /* now we determine if the current line is "new" */ /* new == no documentation for 16 hours */ + CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', charttime) - DATE_TRUNC('hour', charttime_lag)) / 3600 AS BIGINT) AS central_line_duration, /* now we determine if the current line is "new" */ /* new == no documentation for 16 hours */ CASE WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', charttime) - DATE_TRUNC('hour', charttime_lag)) / 3600 AS BIGINT) > 16 THEN 1 diff --git a/mimic-iii/concepts_postgres/organfailure/kdigo_stages.sql b/mimic-iii/concepts_postgres/organfailure/kdigo_stages.sql index 4f8a985f..bbdbd549 100644 --- a/mimic-iii/concepts_postgres/organfailure/kdigo_stages.sql +++ b/mimic-iii/concepts_postgres/organfailure/kdigo_stages.sql @@ -12,11 +12,14 @@ WITH cr_stg AS ( ) THEN 3 WHEN cr.creat >= 4 - -- For patients reaching Stage 3 by SCr >4.0 mg/dl - -- require that the patient first achieve ... acute increase >= 0.3 within 48 hr - -- *or* an increase of >= 1.5 times baseline - and (cr.creat >= (cr.creat_low_past_48hr+0.3) OR cr.creat >= (1.5*cr.creat_low_past_7day)) - then 3 + AND /* For patients reaching Stage 3 by SCr >4.0 mg/dl */ /* require that the patient first achieve ... acute increase >= 0.3 within 48 hr */ /* *or* an increase of >= 1.5 times baseline */ ( + cr.creat >= ( + cr.creat_low_past_48hr + 0.3 + ) + OR cr.creat >= ( + 1.5 * cr.creat_low_past_7day + ) + ) THEN 3 WHEN cr.creat >= ( cr.creat_low_past_7day * 2.0 diff --git a/mimic-iii/notebooks/aline-aws/README.md b/mimic-iii/notebooks/aline-aws/README.md index ecfc4083..526126d3 100644 --- a/mimic-iii/notebooks/aline-aws/README.md +++ b/mimic-iii/notebooks/aline-aws/README.md @@ -17,7 +17,7 @@ You can learn more about the details of this modification and see a performance The study can be reproduced by: -1. Use the below Launch Stack button to deploy access to the MIMIC-III dataset into your AWS account. This will give you real-time access to the MIMIC-III data in your AWS account without having to download a copy of the MIMIC-III dataset. It will also deploy a Jupyter Notebook with access to the content of this GitHub repository in your AWS account. Prior to launching this, please login to the [MIMIC PhysioNet website](https://mimic.physionet.org/), [input your AWS account number](https://physionet.org/settings/cloud/), and [request access to the MIMIC-III Clinical Database on AWS](https://physionet.org/projects/mimiciii/1.4/request_access/2). +1. Use the below Launch Stack button to deploy access to the MIMIC-III dataset into your AWS account. This will give you real-time access to the MIMIC-III data in your AWS account without having to download a copy of the MIMIC-III dataset. It will also deploy a Jupyter Notebook with access to the content of this GitHub repository in your AWS account. Prior to launching this, please login to the [MIMIC PhysioNet website](https://mimic.mit.edu/), [input your AWS account number](https://physionet.org/settings/cloud/), and [request access to the MIMIC-III Clinical Database on AWS](https://physionet.org/projects/mimiciii/1.4/request_access/2). To start this deployment, click the Launch Stack button. On the first screen, the template link has already been specified, so just click next. On the second screen, provide a Stack name (letters and numbers) and click next, on the third screen, just click next. On the forth screen, at the bottom, there is a box that says **I acknowledge that AWS CloudFormation might create IAM resources.**. Check that box, and then click **Create**. Once the Stack has complete deploying, look at the **Outputs** tab of the AWS CloudFormation console for links to your Juypter Notebooks instance. diff --git a/mimic-iii/notebooks/aline-aws/aline-awsathena.ipynb b/mimic-iii/notebooks/aline-aws/aline-awsathena.ipynb index f2c8d0a7..68cf1100 100644 --- a/mimic-iii/notebooks/aline-aws/aline-awsathena.ipynb +++ b/mimic-iii/notebooks/aline-aws/aline-awsathena.ipynb @@ -1122,7 +1122,7 @@ " -- severity of illness just before ventilation\n", " , so.sofa as sofa_first\n", "\n", - " -- vital sign value just preceeding ventilation\n", + " -- vital sign value just preceding ventilation\n", " , vi.map as map_first\n", " , vi.heartrate as hr_first\n", " , vi.temperature as temp_first\n", diff --git a/mimic-iii/notebooks/aline-aws/aline_labs-awsathena.sql b/mimic-iii/notebooks/aline-aws/aline_labs-awsathena.sql index 3273c0c0..ddf11e16 100644 --- a/mimic-iii/notebooks/aline-aws/aline_labs-awsathena.sql +++ b/mimic-iii/notebooks/aline-aws/aline_labs-awsathena.sql @@ -1,6 +1,6 @@ CREATE TABLE DATABASE.ALINE_LABS as -with labs_preceeding as +with labs_preceding as ( select co.icustay_id , l.valuenum, l.charttime @@ -49,7 +49,7 @@ with labs_preceeding as select icustay_id, valuenum, label, obs_after_vent , ROW_NUMBER() over (partition by icustay_id, label, obs_after_vent order by charttime DESC) as rn - from labs_preceeding + from labs_preceding ) , labs_grp as ( diff --git a/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql b/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql index 7aadd4da..bd0ebcab 100644 --- a/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql +++ b/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql @@ -9,13 +9,13 @@ with vitals_stg0 as , case -- MAP, Temperature, HR, CVP, SpO2, when itemid in (456,52,6702,443,220052,220181,225312) then 'MAP' - when itemid in (223762,676,223761,678,226329,227632,227634) then 'Temperature' + when itemid in (223762,676,223761,678) then 'Temperature' when itemid in (211,220045) then 'HeartRate' when itemid in (646,220277) then 'SpO2' else null end as label , case when itemid in (223761,678) and ((valuenum-32)/1.8)<10 then null - when itemid in (223762,676,226329,227632,227634) and valuenum < 10 then null + when itemid in (223762,676) and valuenum < 10 then null -- convert F to C when itemid in (223761,678) then (valuenum-32)/1.8 -- sanity checks on data - one outliter with spo2 < 25 @@ -31,14 +31,13 @@ with vitals_stg0 as ( 456,52,6702,443,220052,220181,225312 -- map , 223762,676,223761,678 -- temp - , 226329,227632,227634 -- CCO / Arctic Sun temp (C) , 211,220045 -- hr , 646,220277 -- spo2 ) and valuenum is not null and coalesce(error,0) != 1 ) --- next, assign an integer where rn=1 is the vital sign just preceeding vent +-- next, assign an integer where rn=1 is the vital sign just preceding vent , vitals_stg1 as ( select diff --git a/mimic-iii/notebooks/aline/README.md b/mimic-iii/notebooks/aline/README.md index 1017a1e2..b817cea6 100644 --- a/mimic-iii/notebooks/aline/README.md +++ b/mimic-iii/notebooks/aline/README.md @@ -10,7 +10,7 @@ The code here reproduces this study in the MIMIC-III database. This involved man # Requirements -There are a number of prerequesites to running this code: +There are a number of prerequisites to running this code: * an installation of MIMIC-III in a PostgreSQL database * Python 2.7 with the numpy, pandas, matplotlib, and psycopg2 packages diff --git a/mimic-iii/notebooks/aline/aline.ipynb b/mimic-iii/notebooks/aline/aline.ipynb index a0c1df3f..bdc92212 100644 --- a/mimic-iii/notebooks/aline/aline.ipynb +++ b/mimic-iii/notebooks/aline/aline.ipynb @@ -1062,7 +1062,7 @@ " -- severity of illness just before ventilation\n", " , so.sofa as sofa_first\n", "\n", - " -- vital sign value just preceeding ventilation\n", + " -- vital sign value just preceding ventilation\n", " , vi.map as map_first\n", " , vi.heartrate as hr_first\n", " , vi.temperature as temp_first\n", diff --git a/mimic-iii/notebooks/aline/aline_labs.sql b/mimic-iii/notebooks/aline/aline_labs.sql index f7efec0a..1f07ac4b 100644 --- a/mimic-iii/notebooks/aline/aline_labs.sql +++ b/mimic-iii/notebooks/aline/aline_labs.sql @@ -3,7 +3,7 @@ DROP MATERIALIZED VIEW IF EXISTS ALINE_LABS CASCADE; CREATE MATERIALIZED VIEW ALINE_LABS as -with labs_preceeding as +with labs_preceding as ( select co.icustay_id , l.valuenum, l.charttime @@ -52,7 +52,7 @@ with labs_preceeding as select icustay_id, valuenum, label, obs_after_vent , ROW_NUMBER() over (partition by icustay_id, label, obs_after_vent order by charttime DESC) as rn - from labs_preceeding + from labs_preceding ) , labs_grp as ( diff --git a/mimic-iii/notebooks/aline/aline_vitals.sql b/mimic-iii/notebooks/aline/aline_vitals.sql index 033be2c6..f3f93a2c 100644 --- a/mimic-iii/notebooks/aline/aline_vitals.sql +++ b/mimic-iii/notebooks/aline/aline_vitals.sql @@ -10,13 +10,13 @@ with vitals_stg0 as , case -- MAP, Temperature, HR, CVP, SpO2, when itemid in (456,52,6702,443,220052,220181,225312) then 'MAP' - when itemid in (223762,676,223761,678,226329,227632,227634) then 'Temperature' + when itemid in (223762,676,223761,678) then 'Temperature' when itemid in (211,220045) then 'HeartRate' when itemid in (646,220277) then 'SpO2' else null end as label , case when itemid in (223761,678) and ((valuenum-32)/1.8)<10 then null - when itemid in (223762,676,226329,227632,227634) and valuenum < 10 then null + when itemid in (223762,676) and valuenum < 10 then null -- convert F to C when itemid in (223761,678) then (valuenum-32)/1.8 -- sanity checks on data - one outliter with spo2 < 25 @@ -32,14 +32,13 @@ with vitals_stg0 as ( 456,52,6702,443,220052,220181,225312 -- map , 223762,676,223761,678 -- temp - , 226329,227632,227634 -- CCO / Arctic Sun temp (C) , 211,220045 -- hr , 646,220277 -- spo2 ) and valuenum is not null and coalesce(error,0) != 1 ) --- next, assign an integer where rn=1 is the vital sign just preceeding vent +-- next, assign an integer where rn=1 is the vital sign just preceding vent , vitals_stg1 as ( select diff --git a/mimic-iv/concepts/README.md b/mimic-iv/concepts/README.md index b2c3bd48..f2c55041 100644 --- a/mimic-iv/concepts/README.md +++ b/mimic-iv/concepts/README.md @@ -1,11 +1,11 @@ # MIMIC-IV Concepts This folder contains scripts to generate useful abstractions of raw MIMIC-IV data ("concepts"). -The scripts are written using the **BigQuery Standard SQL Dialect**. Concepts are categorized into folders if possible, otherwise they remain in the top-level directory. The [postgres](/mimic-iv/concepts_postgres) subfolder contains automatically generated PostgreSQL versions of these scripts; [see below for how these were generated](#postgresql-concepts). Concepts are categorized into folders if possible, otherwise they remain in the top-level directory. +The scripts are written using the **BigQuery Standard SQL Dialect**. Concepts are categorized into folders if possible, otherwise they remain in the top-level directory. The [concepts_postgres](/mimic-iv/concepts_postgres) and [concepts_duckdb](/mimic-iv/concepts_duckdb) folders contain automatically generated PostgreSQL and DuckDB versions of these scripts; see the [top-level readme](/mimic-iv/README.md#generating-the-concepts) for how these were generated. The concepts are organized into individual SQL scripts, with each script generating a table. The BigQuery `mimiciv_derived` dataset under `physionet-data` contains the concepts pregenerated. Access to this dataset is available to MIMIC-IV approved users: see the [cloud instructions](https://mimic.mit.edu/docs/gettingstarted/cloud/) on how to access MIMIC-IV on BigQuery (which includes the derived concepts). -See the [top-level readme](mimic-iv/README.md) for more information about generating the concepts. +See the [top-level readme](/mimic-iv/README.md) for more information about generating the concepts. ## Concept Index diff --git a/mimic-iv/concepts/comorbidity/charlson.sql b/mimic-iv/concepts/comorbidity/charlson.sql index cb6c168c..15911b37 100644 --- a/mimic-iv/concepts/comorbidity/charlson.sql +++ b/mimic-iv/concepts/comorbidity/charlson.sql @@ -285,8 +285,6 @@ WITH diag AS ( -- Any malignancy, including lymphoma and leukemia, -- except malignant neoplasm of skin. - -- Exclude ICD-10-CM-only C4A (Merkel); Quan 2005 is WHO ICD-10 (#2017). - -- C7A/C7B sit outside these ranges and are also left excluded. , MAX(CASE WHEN SUBSTR(icd9_code, 1, 3) BETWEEN '140' AND '172' OR @@ -304,10 +302,7 @@ WITH diag AS ( OR SUBSTR(icd10_code, 1, 3) BETWEEN 'C37' AND 'C41' OR - ( - SUBSTR(icd10_code, 1, 3) BETWEEN 'C45' AND 'C58' - AND SUBSTR(icd10_code, 1, 3) != 'C4A' - ) + SUBSTR(icd10_code, 1, 3) BETWEEN 'C45' AND 'C58' OR SUBSTR(icd10_code, 1, 3) BETWEEN 'C60' AND 'C76' OR @@ -355,11 +350,10 @@ WITH diag AS ( SELECT hadm_id , age - -- CCI age bands are [50,60), [60,70), [70,80), 80+ (not <= decade). - , CASE WHEN age < 50 THEN 0 - WHEN age < 60 THEN 1 - WHEN age < 70 THEN 2 - WHEN age < 80 THEN 3 + , CASE WHEN age <= 50 THEN 0 + WHEN age <= 60 THEN 1 + WHEN age <= 70 THEN 2 + WHEN age <= 80 THEN 3 ELSE 4 END AS age_score FROM `physionet-data.mimiciv_derived.age` ) diff --git a/mimic-iv/concepts/firstday/first_day_sofa.sql b/mimic-iv/concepts/firstday/first_day_sofa.sql index ffd3d3c2..62282af7 100644 --- a/mimic-iv/concepts/firstday/first_day_sofa.sql +++ b/mimic-iv/concepts/firstday/first_day_sofa.sql @@ -196,8 +196,8 @@ WITH vaso_stg AS ( OR rate_norepinephrine > 0.1 THEN 4 WHEN rate_dopamine > 5 - OR (rate_epinephrine > 0 AND rate_epinephrine <= 0.1) - OR (rate_norepinephrine > 0 AND rate_norepinephrine <= 0.1) + OR rate_epinephrine <= 0.1 + OR rate_norepinephrine <= 0.1 THEN 3 WHEN rate_dopamine > 0 OR rate_dobutamine > 0 THEN 2 WHEN mbp_min < 70 THEN 1 diff --git a/mimic-iv/concepts/measurement/oxygen_delivery.sql b/mimic-iv/concepts/measurement/oxygen_delivery.sql index 9c582c36..e76f6303 100644 --- a/mimic-iv/concepts/measurement/oxygen_delivery.sql +++ b/mimic-iv/concepts/measurement/oxygen_delivery.sql @@ -89,9 +89,8 @@ WITH ce_stg1 AS ( FULL OUTER JOIN o2 ON ce.subject_id = o2.subject_id AND ce.charttime = o2.charttime - -- limit to 1 row per subject_id/charttime/itemid from ce_stg2; - -- keep o2-only charttimes (ce side NULL after the full outer join) - WHERE ce.rn = 1 OR ce.subject_id IS NULL + -- limit to 1 row per subject_id/charttime/itemid from ce_stg2 + WHERE ce.rn = 1 ) SELECT diff --git a/mimic-iv/concepts/measurement/urine_output_rate.sql b/mimic-iv/concepts/measurement/urine_output_rate.sql index bc674014..01939211 100644 --- a/mimic-iv/concepts/measurement/urine_output_rate.sql +++ b/mimic-iv/concepts/measurement/urine_output_rate.sql @@ -16,14 +16,12 @@ WITH tm AS ( ) -- now calculate time since last UO measurement --- Use SECOND diffs (like kdigo_uo) so BigQuery and PostgreSQL agree (#1549). --- BigQuery DATETIME_DIFF(..., MINUTE/HOUR) truncates; SECOND keeps fractions. , uo_tm AS ( SELECT tm.stay_id , CASE WHEN LAG(charttime) OVER w IS NULL - THEN DATETIME_DIFF(charttime, intime_hr, SECOND) / 60.0 - ELSE DATETIME_DIFF(charttime, LAG(charttime) OVER w, SECOND) / 60.0 + THEN DATETIME_DIFF(charttime, intime_hr, MINUTE) + ELSE DATETIME_DIFF(charttime, LAG(charttime) OVER w, MINUTE) END AS tm_since_last_uo , uo.charttime , uo.urineoutput @@ -47,33 +45,24 @@ WITH tm AS ( -- to 1 hour of UO, therefore we use '5' and '11' to restrict the -- period, rather than 6/12 this assumption may overestimate UO rate -- when documentation is done less than hourly - , SUM(CASE - WHEN DATETIME_DIFF( - io.charttime, iosum.charttime, SECOND - ) / 3600.0 <= 5 + , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_6hr , ROUND(CAST( SUM( CASE - WHEN DATETIME_DIFF( - io.charttime, iosum.charttime, SECOND - ) / 3600.0 <= 5 + WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 THEN iosum.tm_since_last_uo ELSE NULL END) / 60.0 AS NUMERIC ), 6) AS uo_tm_6hr - , SUM(CASE - WHEN DATETIME_DIFF( - io.charttime, iosum.charttime, SECOND - ) / 3600.0 <= 11 + , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_12hr , ROUND(CAST( SUM( CASE - WHEN DATETIME_DIFF( - io.charttime, iosum.charttime, SECOND - ) / 3600.0 <= 11 + WHEN + DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 THEN iosum.tm_since_last_uo ELSE NULL END) / 60.0 AS NUMERIC ), 6) AS uo_tm_12hr diff --git a/mimic-iv/concepts/measurement/vitalsign.sql b/mimic-iv/concepts/measurement/vitalsign.sql index c2899b4d..030e4055 100644 --- a/mimic-iv/concepts/measurement/vitalsign.sql +++ b/mimic-iv/concepts/measurement/vitalsign.sql @@ -53,7 +53,7 @@ SELECT AND valuenum < 120 THEN (valuenum - 32) / 1.8 -- already in degC, no conversion necessary - WHEN itemid IN (223762, 226329, 227632, 227634) + WHEN itemid IN (223762) AND valuenum > 10 AND valuenum < 50 THEN valuenum END) @@ -91,9 +91,7 @@ WHERE ce.stay_id IS NOT NULL , 220621 -- Glucose (serum) , 226537 -- Glucose (whole blood) -- TEMPERATURE - , 226329 -- Blood Temperature CCO (C) - , 227632 -- Arctic Sun/Alsius Temp # 1 C - , 227634 -- Arctic Sun/Alsius Temp # 2 C + -- 226329 -- Blood Temperature CCO (C) , 223762 -- "Temperature Celsius" , 223761 -- "Temperature Fahrenheit" , 224642 -- Temperature Site diff --git a/mimic-iv/concepts/score/lods.sql b/mimic-iv/concepts/score/lods.sql index 1bde1f88..bc105402 100644 --- a/mimic-iv/concepts/score/lods.sql +++ b/mimic-iv/concepts/score/lods.sql @@ -168,8 +168,7 @@ WITH cpap AS ( WHEN urineoutput >= 10000.0 THEN 3 WHEN creatinine_max >= 1.20 THEN 1 WHEN bun_max >= 17.0 THEN 1 - -- note: do not use bun_max >= 7.50; that is a mmol/L cutpoint - -- incorrectly applied to MIMIC BUN in mg/dL (#1065) + WHEN bun_max >= 7.50 THEN 1 ELSE 0 END AS renal diff --git a/mimic-iv/concepts/score/sofa.sql b/mimic-iv/concepts/score/sofa.sql index 049b3920..0dbf28d1 100644 --- a/mimic-iv/concepts/score/sofa.sql +++ b/mimic-iv/concepts/score/sofa.sql @@ -280,8 +280,8 @@ WITH co AS ( OR rate_norepinephrine > 0.1 THEN 4 WHEN rate_dopamine > 5 - OR (rate_epinephrine > 0 AND rate_epinephrine <= 0.1) - OR (rate_norepinephrine > 0 AND rate_norepinephrine <= 0.1) + OR rate_epinephrine <= 0.1 + OR rate_norepinephrine <= 0.1 THEN 3 WHEN rate_dopamine > 0 OR rate_dobutamine > 0 diff --git a/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql b/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql index 682a6f5c..9341d3a4 100644 --- a/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql +++ b/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql @@ -8,7 +8,7 @@ WITH all_hours AS ( THEN it.intime_hr ELSE DATE_TRUNC('HOUR', CAST(it.intime_hr AS TIMESTAMP)) + INTERVAL '1' HOUR END AS endtime, - (SELECT list(g) FROM generate_series(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS INT)) AS t(g)) AS hrs + GENERATE_SERIES(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS INT)) AS hrs FROM mimiciv_derived.icustay_times AS it ) SELECT From 73befc05eb3eb099611b9796f096d5ca7b8a8960 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sat, 25 Jul 2026 08:56:57 +0300 Subject: [PATCH 49/49] chore: sync PR head Co-authored-by: Cursor