diff --git a/.github/workflows/lint_sqlfluff.yml b/.github/workflows/lint_sqlfluff.yml index 9401198e0..ca4c8c8bb 100644 --- a/.github/workflows/lint_sqlfluff.yml +++ b/.github/workflows/lint_sqlfluff.yml @@ -35,8 +35,12 @@ jobs: shell: bash run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json - name: Annotate + # Fork PRs cannot create check runs with GITHUB_TOKEN; lint already ran. + continue-on-error: true + if: steps.get_files_to_lint.outputs.lintees != '' uses: yuzutech/annotations-action@v0.6.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" title: "SQLFluff Lint" - input: "./annotations.json" \ No newline at end of file + input: "./annotations.json" + ignore-missing-file: true \ No newline at end of file diff --git a/mimic-iii/buildmimic/duckdb/import_duckdb.sh b/mimic-iii/buildmimic/duckdb/import_duckdb.sh index 15d4977fe..badf83b86 100755 --- a/mimic-iii/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iii/buildmimic/duckdb/import_duckdb.sh @@ -84,9 +84,11 @@ make_table_name () { # load data into database find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do make_table_name "$FILE" - echo "Loading $FILE .. \c" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") + printf "Loading %s .. " "$FILE" try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL echo "done!" done && echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iii/buildmimic/postgres/Makefile b/mimic-iii/buildmimic/postgres/Makefile index 17ea42e20..29953adbd 100644 --- a/mimic-iii/buildmimic/postgres/Makefile +++ b/mimic-iii/buildmimic/postgres/Makefile @@ -104,7 +104,7 @@ mimic-build-gz: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v mimic_data_dir=${datadir} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -151,7 +151,7 @@ mimic-build: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v mimic_data_dir=${DATADIR} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -184,7 +184,7 @@ ifeq ("$(physionetuser)","") @echo 'Call the makefile again with physionetuser=' @echo ' e.g. make eicu-download datadir=/path/to/data physionetuser=hello@physionet.org' else - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" endif mimic-demo-download: @@ -192,7 +192,7 @@ mimic-demo-download: @echo '-- Downloading MIMIC-III from PhysioNet --' @echo '------------------------------------------' @echo '' - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" #This is fairly inelegant and could be tidied with a for loop and an if to check for gzip, #but need to maintain compatibility with Windows, which baffling lacks these things diff --git a/mimic-iii/buildmimic/postgres/create_mimic_user.sh b/mimic-iii/buildmimic/postgres/create_mimic_user.sh index 772764ce7..d639f5fcc 100755 --- a/mimic-iii/buildmimic/postgres/create_mimic_user.sh +++ b/mimic-iii/buildmimic/postgres/create_mimic_user.sh @@ -20,6 +20,13 @@ else echo "User is set to '$MIMIC_USER'"; fi +# escape ' in password for SQL string literal +MIMIC_PASSWORD_SQL=$(printf '%s' "$MIMIC_PASSWORD" | sed "s/'/''/g") +# quote SQL identifiers (double any embedded ") +sql_ident () { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; } +MIMIC_USER_SQL=$(sql_ident "$MIMIC_USER") +MIMIC_DB_SQL=$(sql_ident "$MIMIC_DB") + PSQL='psql' # add in the host/port, if they were specified (not null, -n) @@ -58,11 +65,11 @@ fi if [ "$MIMIC_USER" != "postgres" ]; then # we need to create this user via postgres # use SUDO to login as postgres - $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER; CREATE USER $MIMIC_USER WITH PASSWORD '$MIMIC_PASSWORD';" + $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER_SQL; CREATE USER $MIMIC_USER_SQL WITH PASSWORD '$MIMIC_PASSWORD_SQL';" fi if [ "$MIMIC_DB" != "postgres" ]; then # drop and recreate the database - $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB;" - $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB OWNER $MIMIC_USER;" -fi \ No newline at end of file + $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB_SQL;" + $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB_SQL OWNER $MIMIC_USER_SQL;" +fi diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index 801651c7a..4e0113d35 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -10,6 +10,34 @@ CHUNKSIZE = 10 ** 6 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME) +# Column dtypes for tables that trigger pandas mixed-type warnings when +# loaded with the default low_memory chunked inference (see #1237). +# Types mirror mimic-iii/buildmimic/postgres/postgres_create_tables.sql. +TABLE_DTYPES = { + "chartevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "datetimeevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "inputevents_cv": { + "ORIGINALRATE": "float64", + "ORIGINALRATEUOM": "string", + "ORIGINALSITE": "string", + }, + "noteevents": { + "CHARTTIME": "string", + "STORETIME": "string", + "ISERROR": "string", + }, +} + def _table_name_from_csv(filename: str) -> str: """Derive SQL table name from a CSV path (literal suffix, not str.strip).""" @@ -21,20 +49,39 @@ def _table_name_from_csv(filename: str) -> str: return name.lower() +def _read_csv(path, table, **kwargs): + # low_memory=False avoids dtype re-inference across chunks; table-specific + # dtypes pin the columns that otherwise flip between numeric/object. + return pd.read_csv( + path, + index_col="ROW_ID", + low_memory=False, + dtype=TABLE_DTYPES.get(table), + **kwargs, + ) + + if os.path.exists(DATABASE_NAME): msg = "File {} already exists.".format(DATABASE_NAME) print(msg) sys.exit() +# Prefer .csv.gz when both exist for the same table; also load plain .csv +# (import.sh already accepts both). +files_by_table = {} +for f in glob("*.csv"): + files_by_table[_table_name_from_csv(f)] = f for f in glob("*.csv.gz"): + files_by_table[_table_name_from_csv(f)] = f + +for table, f in sorted(files_by_table.items()): print("Starting processing {}".format(f)) - table = _table_name_from_csv(f) if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, index_col="ROW_ID") + df = _read_csv(f, table) df.to_sql(table, CONNECTION_STRING) else: # If the file is too large, let's do the work in chunks - for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE): + for chunk in _read_csv(f, table, chunksize=CHUNKSIZE): chunk.to_sql(table, CONNECTION_STRING, if_exists="append") print("Finished processing {}".format(f)) diff --git a/mimic-iii/buildmimic/sqlite/import.sh b/mimic-iii/buildmimic/sqlite/import.sh index dbf6b9ce4..58b85abb4 100755 --- a/mimic-iii/buildmimic/sqlite/import.sh +++ b/mimic-iii/buildmimic/sqlite/import.sh @@ -28,11 +28,11 @@ for FILE in *; do TABLE_NAME=$(echo "${FILE%%.*}" | tr "[:upper:]" "[:lower:]") case "$FILE" in *csv) - IMPORT_CMD=".import $FILE $TABLE_NAME" + IMPORT_CMD=".import \"$FILE\" $TABLE_NAME" ;; # need to decompress csv before load *csv.gz) - IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME" + IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME" ;; # not a data file so skip *) @@ -40,7 +40,7 @@ for FILE in *; do ;; esac echo "Loading $FILE." - sqlite3 $OUTFILE < 16 THEN 1 ELSE 0 END AS arterial_line_new FROM cv0 ), cv2 AS ( diff --git a/mimic-iii/concepts_duckdb/durations/central_line_durations.sql b/mimic-iii/concepts_duckdb/durations/central_line_durations.sql index d31940e02..33da694ac 100644 --- a/mimic-iii/concepts_duckdb/durations/central_line_durations.sql +++ b/mimic-iii/concepts_duckdb/durations/central_line_durations.sql @@ -200,7 +200,7 @@ WITH mv AS ( icustay_id, charttime, charttime_lag, - charttime - charttime_lag AS central_line_duration, + DATE_DIFF('HOUR', charttime_lag, charttime) AS central_line_duration, CASE WHEN DATE_DIFF('HOUR', charttime_lag, charttime) > 16 THEN 1 ELSE 0 END AS central_line_new FROM cv0 ), cv2 AS ( diff --git a/mimic-iii/concepts_duckdb/organfailure/kdigo_stages.sql b/mimic-iii/concepts_duckdb/organfailure/kdigo_stages.sql index 8db780654..83cdd7cb4 100644 --- a/mimic-iii/concepts_duckdb/organfailure/kdigo_stages.sql +++ b/mimic-iii/concepts_duckdb/organfailure/kdigo_stages.sql @@ -12,7 +12,10 @@ WITH cr_stg AS ( THEN 3 WHEN cr.creat >= 4 AND ( - cr.creat_low_past_48hr <= 3.7 OR cr.creat >= ( + cr.creat >= ( + cr.creat_low_past_48hr + 0.3 + ) + OR cr.creat >= ( 1.5 * cr.creat_low_past_7day ) ) diff --git a/mimic-iii/concepts_postgres/README.md b/mimic-iii/concepts_postgres/README.md index 250bf012c..0510e718c 100644 --- a/mimic-iii/concepts_postgres/README.md +++ b/mimic-iii/concepts_postgres/README.md @@ -6,7 +6,7 @@ If you would like to contribute a correction, do not make it here. Instead, make Two files are hand-written for PostgreSQL and have no BigQuery source: -* [diagnosis/ccs_multi_dx.sql](diagnosis/ccs_multi_dx.sql) loads the ICD-9 to CCS mapping from [diagnosis/ccs_multi_dx.csv.gz](diagnosis/ccs_multi_dx.csv.gz). +* [diagnosis/ccs_multi_dx.sql](diagnosis/ccs_multi_dx.sql) loads the ICD-9 to CCS mapping from [ccs_multi_dx.csv.gz in the concepts folder](/mimic-iii/concepts/diagnosis/ccs_multi_dx.csv.gz). * [demographics/note_counts.sql](demographics/note_counts.sql) is an optional PostgreSQL-only concept which summarizes note counts per hospital admission (it is not run by the make script). ## Using these concepts diff --git a/mimic-iii/concepts_postgres/diagnosis/ccs_dx.sql b/mimic-iii/concepts_postgres/diagnosis/ccs_dx.sql index 84215817e..19c352b05 100644 --- a/mimic-iii/concepts_postgres/diagnosis/ccs_dx.sql +++ b/mimic-iii/concepts_postgres/diagnosis/ccs_dx.sql @@ -1,6 +1,6 @@ -- THIS SCRIPT IS AUTOMATICALLY GENERATED. DO NOT EDIT IT DIRECTLY. DROP TABLE IF EXISTS mimiciii_derived.ccs_dx; CREATE TABLE mimiciii_derived.ccs_dx AS -/* add in matched ID, name, and ccs_id */ /* matched id (ccs_mid): the ccs ID with the hierachy, e.g. 7.1.2.1 */ /* name (ccs_name): the most granular CCS category the diagnosis is in */ /* ID (ccs_id): the CCS identifier for the ICD-9 code (integer) */ +/* add in matched ID, name, and ccs_id */ /* matched id (ccs_mid): the ccs ID with the hierarchy, e.g. 7.1.2.1 */ /* name (ccs_name): the most granular CCS category the diagnosis is in */ /* ID (ccs_id): the CCS identifier for the ICD-9 code (integer) */ SELECT icd9_code, COALESCE(ccs_level4, ccs_level3, ccs_level2, ccs_level1) AS ccs_matched_id, /* remove the trailing ccs_id from name column, i.e. "Burns [240.]" -> "Burns" */ diff --git a/mimic-iii/concepts_postgres/diagnosis/ccs_multi_dx.sql b/mimic-iii/concepts_postgres/diagnosis/ccs_multi_dx.sql index dd30e9c4c..22a09d48a 100644 --- a/mimic-iii/concepts_postgres/diagnosis/ccs_multi_dx.sql +++ b/mimic-iii/concepts_postgres/diagnosis/ccs_multi_dx.sql @@ -1,6 +1,7 @@ -- There is no BigQuery source for this script. --- Loads the CCS multi-level diagnosis mapping from the CSV distributed --- alongside this script. On BigQuery the same table is loaded with: +-- Loads the CCS multi-level diagnosis mapping from the CSV stored in the +-- concepts folder (../concepts/diagnosis/ccs_multi_dx.csv.gz). On BigQuery +-- the same table is loaded with: -- bq load mimiciii_derived.ccs_multi_dx diagnosis/ccs_multi_dx.csv.gz diagnosis/ccs_multi_dx.json -- Run from the concepts_postgres directory so the relative path resolves. DROP TABLE IF EXISTS mimiciii_derived.ccs_multi_dx; @@ -17,4 +18,4 @@ CREATE TABLE mimiciii_derived.ccs_multi_dx ccs_level4 VARCHAR(10), ccs_group4 VARCHAR(100) ); -\COPY mimiciii_derived.ccs_multi_dx FROM PROGRAM 'gzip -dc diagnosis/ccs_multi_dx.csv.gz' CSV HEADER; +\COPY mimiciii_derived.ccs_multi_dx FROM PROGRAM 'gzip -dc ../concepts/diagnosis/ccs_multi_dx.csv.gz' CSV HEADER; diff --git a/mimic-iii/concepts_postgres/durations/arterial_line_durations.sql b/mimic-iii/concepts_postgres/durations/arterial_line_durations.sql index 992809e35..cc40408b6 100644 --- a/mimic-iii/concepts_postgres/durations/arterial_line_durations.sql +++ b/mimic-iii/concepts_postgres/durations/arterial_line_durations.sql @@ -111,7 +111,7 @@ WITH mv AS ( icustay_id, charttime, charttime_lag, /* if the current observation indicates a line is present */ /* calculate the time since the last charted line */ - charttime - charttime_lag AS arterial_line_duration, /* now we determine if the current line is "new" */ /* new == no documentation for 16 hours */ + CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', charttime) - DATE_TRUNC('hour', charttime_lag)) / 3600 AS BIGINT) AS arterial_line_duration, /* now we determine if the current line is "new" */ /* new == no documentation for 16 hours */ CASE WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', charttime) - DATE_TRUNC('hour', charttime_lag)) / 3600 AS BIGINT) > 16 THEN 1 diff --git a/mimic-iii/concepts_postgres/durations/central_line_durations.sql b/mimic-iii/concepts_postgres/durations/central_line_durations.sql index b1b6ef2c5..905e17a0f 100644 --- a/mimic-iii/concepts_postgres/durations/central_line_durations.sql +++ b/mimic-iii/concepts_postgres/durations/central_line_durations.sql @@ -201,7 +201,7 @@ WITH mv AS ( icustay_id, charttime, charttime_lag, /* if the current observation indicates a line is present */ /* calculate the time since the last charted line */ - charttime - charttime_lag AS central_line_duration, /* now we determine if the current line is "new" */ /* new == no documentation for 16 hours */ + CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', charttime) - DATE_TRUNC('hour', charttime_lag)) / 3600 AS BIGINT) AS central_line_duration, /* now we determine if the current line is "new" */ /* new == no documentation for 16 hours */ CASE WHEN CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', charttime) - DATE_TRUNC('hour', charttime_lag)) / 3600 AS BIGINT) > 16 THEN 1 diff --git a/mimic-iii/concepts_postgres/organfailure/kdigo_stages.sql b/mimic-iii/concepts_postgres/organfailure/kdigo_stages.sql index 4f8a985fd..bbdbd5495 100644 --- a/mimic-iii/concepts_postgres/organfailure/kdigo_stages.sql +++ b/mimic-iii/concepts_postgres/organfailure/kdigo_stages.sql @@ -12,11 +12,14 @@ WITH cr_stg AS ( ) THEN 3 WHEN cr.creat >= 4 - -- For patients reaching Stage 3 by SCr >4.0 mg/dl - -- require that the patient first achieve ... acute increase >= 0.3 within 48 hr - -- *or* an increase of >= 1.5 times baseline - and (cr.creat >= (cr.creat_low_past_48hr+0.3) OR cr.creat >= (1.5*cr.creat_low_past_7day)) - then 3 + AND /* For patients reaching Stage 3 by SCr >4.0 mg/dl */ /* require that the patient first achieve ... acute increase >= 0.3 within 48 hr */ /* *or* an increase of >= 1.5 times baseline */ ( + cr.creat >= ( + cr.creat_low_past_48hr + 0.3 + ) + OR cr.creat >= ( + 1.5 * cr.creat_low_past_7day + ) + ) THEN 3 WHEN cr.creat >= ( cr.creat_low_past_7day * 2.0 diff --git a/mimic-iii/concepts_postgres/postgres-make-concepts.sql b/mimic-iii/concepts_postgres/postgres-make-concepts.sql index 3779bef6c..318b3da1c 100644 --- a/mimic-iii/concepts_postgres/postgres-make-concepts.sql +++ b/mimic-iii/concepts_postgres/postgres-make-concepts.sql @@ -61,6 +61,7 @@ SET search_path TO mimiciii_derived, mimiciii; \i pivot/pivoted_uo.sql \i pivot/pivoted_vital.sql \i pivot/pivoted_bg_art.sql +\i pivot/pivoted_oasis.sql \i pivot/pivoted_sofa.sql -- comorbidity diff --git a/mimic-iii/notebooks/aline-aws/README.md b/mimic-iii/notebooks/aline-aws/README.md index ecfc40834..526126d35 100644 --- a/mimic-iii/notebooks/aline-aws/README.md +++ b/mimic-iii/notebooks/aline-aws/README.md @@ -17,7 +17,7 @@ You can learn more about the details of this modification and see a performance The study can be reproduced by: -1. Use the below Launch Stack button to deploy access to the MIMIC-III dataset into your AWS account. This will give you real-time access to the MIMIC-III data in your AWS account without having to download a copy of the MIMIC-III dataset. It will also deploy a Jupyter Notebook with access to the content of this GitHub repository in your AWS account. Prior to launching this, please login to the [MIMIC PhysioNet website](https://mimic.physionet.org/), [input your AWS account number](https://physionet.org/settings/cloud/), and [request access to the MIMIC-III Clinical Database on AWS](https://physionet.org/projects/mimiciii/1.4/request_access/2). +1. Use the below Launch Stack button to deploy access to the MIMIC-III dataset into your AWS account. This will give you real-time access to the MIMIC-III data in your AWS account without having to download a copy of the MIMIC-III dataset. It will also deploy a Jupyter Notebook with access to the content of this GitHub repository in your AWS account. Prior to launching this, please login to the [MIMIC PhysioNet website](https://mimic.mit.edu/), [input your AWS account number](https://physionet.org/settings/cloud/), and [request access to the MIMIC-III Clinical Database on AWS](https://physionet.org/projects/mimiciii/1.4/request_access/2). To start this deployment, click the Launch Stack button. On the first screen, the template link has already been specified, so just click next. On the second screen, provide a Stack name (letters and numbers) and click next, on the third screen, just click next. On the forth screen, at the bottom, there is a box that says **I acknowledge that AWS CloudFormation might create IAM resources.**. Check that box, and then click **Create**. Once the Stack has complete deploying, look at the **Outputs** tab of the AWS CloudFormation console for links to your Juypter Notebooks instance. diff --git a/mimic-iii/notebooks/aline-aws/aline-awsathena.ipynb b/mimic-iii/notebooks/aline-aws/aline-awsathena.ipynb index f2c8d0a72..68cf11005 100644 --- a/mimic-iii/notebooks/aline-aws/aline-awsathena.ipynb +++ b/mimic-iii/notebooks/aline-aws/aline-awsathena.ipynb @@ -1122,7 +1122,7 @@ " -- severity of illness just before ventilation\n", " , so.sofa as sofa_first\n", "\n", - " -- vital sign value just preceeding ventilation\n", + " -- vital sign value just preceding ventilation\n", " , vi.map as map_first\n", " , vi.heartrate as hr_first\n", " , vi.temperature as temp_first\n", diff --git a/mimic-iii/notebooks/aline-aws/aline_labs-awsathena.sql b/mimic-iii/notebooks/aline-aws/aline_labs-awsathena.sql index 3273c0c0e..ddf11e169 100644 --- a/mimic-iii/notebooks/aline-aws/aline_labs-awsathena.sql +++ b/mimic-iii/notebooks/aline-aws/aline_labs-awsathena.sql @@ -1,6 +1,6 @@ CREATE TABLE DATABASE.ALINE_LABS as -with labs_preceeding as +with labs_preceding as ( select co.icustay_id , l.valuenum, l.charttime @@ -49,7 +49,7 @@ with labs_preceeding as select icustay_id, valuenum, label, obs_after_vent , ROW_NUMBER() over (partition by icustay_id, label, obs_after_vent order by charttime DESC) as rn - from labs_preceeding + from labs_preceding ) , labs_grp as ( diff --git a/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql b/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql index 6d30869ca..bd0ebcab5 100644 --- a/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql +++ b/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql @@ -37,7 +37,7 @@ with vitals_stg0 as and valuenum is not null and coalesce(error,0) != 1 ) --- next, assign an integer where rn=1 is the vital sign just preceeding vent +-- next, assign an integer where rn=1 is the vital sign just preceding vent , vitals_stg1 as ( select diff --git a/mimic-iii/notebooks/aline/README.md b/mimic-iii/notebooks/aline/README.md index 1017a1e2f..b817cea66 100644 --- a/mimic-iii/notebooks/aline/README.md +++ b/mimic-iii/notebooks/aline/README.md @@ -10,7 +10,7 @@ The code here reproduces this study in the MIMIC-III database. This involved man # Requirements -There are a number of prerequesites to running this code: +There are a number of prerequisites to running this code: * an installation of MIMIC-III in a PostgreSQL database * Python 2.7 with the numpy, pandas, matplotlib, and psycopg2 packages diff --git a/mimic-iii/notebooks/aline/aline.ipynb b/mimic-iii/notebooks/aline/aline.ipynb index a0c1df3f8..bdc92212d 100644 --- a/mimic-iii/notebooks/aline/aline.ipynb +++ b/mimic-iii/notebooks/aline/aline.ipynb @@ -1062,7 +1062,7 @@ " -- severity of illness just before ventilation\n", " , so.sofa as sofa_first\n", "\n", - " -- vital sign value just preceeding ventilation\n", + " -- vital sign value just preceding ventilation\n", " , vi.map as map_first\n", " , vi.heartrate as hr_first\n", " , vi.temperature as temp_first\n", diff --git a/mimic-iii/notebooks/aline/aline_labs.sql b/mimic-iii/notebooks/aline/aline_labs.sql index f7efec0a9..1f07ac4b8 100644 --- a/mimic-iii/notebooks/aline/aline_labs.sql +++ b/mimic-iii/notebooks/aline/aline_labs.sql @@ -3,7 +3,7 @@ DROP MATERIALIZED VIEW IF EXISTS ALINE_LABS CASCADE; CREATE MATERIALIZED VIEW ALINE_LABS as -with labs_preceeding as +with labs_preceding as ( select co.icustay_id , l.valuenum, l.charttime @@ -52,7 +52,7 @@ with labs_preceeding as select icustay_id, valuenum, label, obs_after_vent , ROW_NUMBER() over (partition by icustay_id, label, obs_after_vent order by charttime DESC) as rn - from labs_preceeding + from labs_preceding ) , labs_grp as ( diff --git a/mimic-iii/notebooks/aline/aline_vitals.sql b/mimic-iii/notebooks/aline/aline_vitals.sql index 9ee60b797..f3f93a2c0 100644 --- a/mimic-iii/notebooks/aline/aline_vitals.sql +++ b/mimic-iii/notebooks/aline/aline_vitals.sql @@ -38,7 +38,7 @@ with vitals_stg0 as and valuenum is not null and coalesce(error,0) != 1 ) --- next, assign an integer where rn=1 is the vital sign just preceeding vent +-- next, assign an integer where rn=1 is the vital sign just preceding vent , vitals_stg1 as ( select diff --git a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh index 52e34e0ff..f996541a2 100644 --- a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh +++ b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh @@ -22,11 +22,20 @@ fi sleep 2 # loop through each .csv file in the section folder -for fn in `ls $REPORT_PATH`; do - echo `date`: $fn - fn_stem=`echo $fn | cut -d. -f 1` +for fn_path in "$REPORT_PATH"/*.csv; do + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") + echo "$(date): $fn" + fn_stem=${fn%.csv} # run chexpert - must be run from chexpert folder - python $CHEXPERT_PATH/label.py --verbose --reports_path $REPORT_PATH/$fn --output_path ${fn_stem}_labeled.csv --mention_phrases_dir $CHEXPERT_PATH/phrases/mention --unmention_phrases_dir $CHEXPERT_PATH/phrases/unmention --pre_negation_uncertainty_path $CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt --negation_path $CHEXPERT_PATH/patterns/negation.txt --post_negation_uncertainty_path $CHEXPERT_PATH/patterns/post_negation_uncertainty.txt - echo `date`: done! + python "$CHEXPERT_PATH/label.py" --verbose \ + --reports_path "$fn_path" \ + --output_path "${fn_stem}_labeled.csv" \ + --mention_phrases_dir "$CHEXPERT_PATH/phrases/mention" \ + --unmention_phrases_dir "$CHEXPERT_PATH/phrases/unmention" \ + --pre_negation_uncertainty_path "$CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt" \ + --negation_path "$CHEXPERT_PATH/patterns/negation.txt" \ + --post_negation_uncertainty_path "$CHEXPERT_PATH/patterns/post_negation_uncertainty.txt" + echo "$(date): done!" echo '' -done \ No newline at end of file +done diff --git a/mimic-iv-cxr/txt/negbio/run_negbio.sh b/mimic-iv-cxr/txt/negbio/run_negbio.sh index 13cba4795..dab200cd4 100755 --- a/mimic-iv-cxr/txt/negbio/run_negbio.sh +++ b/mimic-iv-cxr/txt/negbio/run_negbio.sh @@ -32,20 +32,22 @@ sleep 2 # mimic_cxr_001.csv # mimic_cxr_002.csv # .. etc -for fn in `ls $BASE_FOLDER`; +for fn_path in "$BASE_FOLDER"/mimic_cxr_*.csv; do + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") echo "Looping through files with mimic_cxr_###.csv pattern." # validate it's a mimic_cxr sections file if [[ $fn =~ ^mimic_cxr_[0-9]+.csv$ ]]; then - export INPUT_FILE=${BASE_FOLDER}/$fn + export INPUT_FILE=$fn_path # remove extension from filename # sets the folder location to stem of original filename # all intermediate files will be saved in this folder export OUTPUT_DIR=${INPUT_FILE::-4} - echo $OUTPUT_DIR - running NegBio.. - python $NEGBIO_PATH/negbio/negbio_csv2bioc.py --output $OUTPUT_DIR/report $INPUT_FILE + echo "$OUTPUT_DIR - running NegBio.." + python "$NEGBIO_PATH/negbio/negbio_csv2bioc.py" --output "$OUTPUT_DIR/report" "$INPUT_FILE" python $NEGBIO_PATH/negbio/negbio_pipeline.py section_split --pattern $NEGBIO_PATH/patterns/section_titles_cxr8.txt --output $OUTPUT_DIR/sections $OUTPUT_DIR/report/* --workers=6 python $NEGBIO_PATH/negbio/negbio_pipeline.py ssplit --output $OUTPUT_DIR/ssplit $OUTPUT_DIR/sections/* --workers=6 python $NEGBIO_PATH/negbio/negbio_pipeline.py parse --output $OUTPUT_DIR/parse $OUTPUT_DIR/ssplit/* --workers=6 @@ -55,7 +57,7 @@ do # ultimate filename we save the labels to export OUTPUT_LABELS=$OUTPUT_DIR/${fn::-4}_labels.csv - python $NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py --phrases_file $NEGBIO_PATH/patterns/chexpert_phrases.yml --output $OUTPUT_LABELS $OUTPUT_DIR/neg/* + python "$NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py" --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_LABELS" "$OUTPUT_DIR"/neg/* fi done echo "Done looping through files." diff --git a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh index 9142f7b40..f6e6a2dad 100644 --- a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh @@ -93,18 +93,29 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-ed # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (ed) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .. " try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected ed/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh index c0425fe6f..487d74050 100644 --- a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh @@ -96,18 +96,23 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-note # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (note) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .." OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -122,5 +127,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected note/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index ab1d37fa3..32f608c48 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -101,18 +101,26 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both uncompressed (.csv) and gzip (.csv.gz). The old '*.csv???' +# pattern only matched names with exactly three chars after ".csv" (i.e. .gz), +# so plain .csv files were silently skipped while the script still reported success. +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv # avoids syntax errors if mimic-iv-ed in the same dir - case $DIRNAME in + case "$DIRNAME" in (hosp|icu) ;; # OK (*) continue; esac - echo "Loading $FILE .. \c" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") + printf "Loading %s .. " "$FILE" OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -127,5 +135,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected hosp/ and icu/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/mysql/validate_demo.sql b/mimic-iv/buildmimic/mysql/validate_demo.sql index 575e84133..ebf4f02dc 100644 --- a/mimic-iv/buildmimic/mysql/validate_demo.sql +++ b/mimic-iv/buildmimic/mysql/validate_demo.sql @@ -31,14 +31,17 @@ FROM ( SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) exp @@ -64,14 +67,17 @@ INNER JOIN SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM procedureevents ) obs diff --git a/mimic-iv/buildmimic/postgres/validate_demo.sql b/mimic-iv/buildmimic/postgres/validate_demo.sql index 303dd98e4..049497e56 100644 --- a/mimic-iv/buildmimic/postgres/validate_demo.sql +++ b/mimic-iv/buildmimic/postgres/validate_demo.sql @@ -22,14 +22,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) @@ -54,14 +57,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM mimiciv_hosp.poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM mimiciv_hosp.prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM mimiciv_hosp.procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM mimiciv_hosp.provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM mimiciv_hosp.services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM mimiciv_hosp.transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM mimiciv_icu.icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM mimiciv_icu.caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM mimiciv_icu.d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.procedureevents ) diff --git a/mimic-iv/buildmimic/sqlite/import.py b/mimic-iv/buildmimic/sqlite/import.py index 68d436479..31b880c44 100644 --- a/mimic-iv/buildmimic/sqlite/import.py +++ b/mimic-iv/buildmimic/sqlite/import.py @@ -110,7 +110,7 @@ def main(): if args.limit > 0: for f in data_files: if 'patients' in f.name: - pt = pd.read_csv(f) + pt = pd.read_csv(f, low_memory=False) break if pt is None: raise FileNotFoundError('Unable to find a patients file in current folder.') @@ -158,7 +158,7 @@ def main(): tablename = tablenames[i] print("Starting processing {}".format(tablename), end='.. ') if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, dtype=mimic_dtypes) + df = pd.read_csv(f, dtype=mimic_dtypes, low_memory=False) df = process_dataframe(df, subjects=subjects) df.to_sql(tablename, connection, index=False) row_counts[tablename] += len(df) diff --git a/mimic-iv/buildmimic/sqlite/import.sh b/mimic-iv/buildmimic/sqlite/import.sh index 08ded2881..fb642d1ce 100755 --- a/mimic-iv/buildmimic/sqlite/import.sh +++ b/mimic-iv/buildmimic/sqlite/import.sh @@ -29,11 +29,11 @@ for FILE in */**.csv*; do TABLE_NAME=$(echo "${BASENAME%%.*}" | tr "[:upper:]" "[:lower:]") case "$FILE" in *csv) - IMPORT_CMD=".import $FILE $TABLE_NAME" + IMPORT_CMD=".import \"$FILE\" $TABLE_NAME" ;; # need to decompress csv before load *csv.gz) - IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME" + IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME" ;; # not a data file so skip *) @@ -41,7 +41,7 @@ for FILE in */**.csv*; do ;; esac echo "Loading $FILE." - sqlite3 $OUTFILE <" exit 1 fi diff --git a/mimic-iv/concepts/make_concepts.sh b/mimic-iv/concepts/make_concepts.sh index 112bcdc22..42be9d225 100644 --- a/mimic-iv/concepts/make_concepts.sh +++ b/mimic-iv/concepts/make_concepts.sh @@ -8,16 +8,15 @@ export MIMIC_VERSION="3.1" # note: max_rows=1 *displays* only one row, but all rows are inserted into the destination table BQ_OPTIONS='--quiet --headless --max_rows=0 --use_legacy_sql=False --replace' -# drop the existing tables in the target dataset -for TABLE in `bq ls physionet-data:${TARGET_DATASET} | cut -d' ' -f3`; -do - # skip the first line of dashes - if [[ "${TABLE:0:2}" == '--' ]]; then - continue - fi +# drop existing tables (CSV format: cut -f3 on pretty ls mis-parses names) +while IFS= read -r TABLE; do + [ -z "${TABLE}" ] && continue echo "Dropping table ${TARGET_DATASET}.${TABLE}" - bq rm -f -q ${TARGET_DATASET}.${TABLE} -done + bq rm -f -q "${TARGET_DATASET}.${TABLE}" +done < <( + bq ls --format=csv --max_results=10000 "physionet-data:${TARGET_DATASET}" \ + | awk -F, 'NR > 1 { print $1 }' +) # create a _version table to store the mimic-iv version, git commit hash, and latest git tag GIT_COMMIT_HASH=$(git rev-parse HEAD) @@ -44,7 +43,7 @@ for table_path in demographics/icustay_times; do table=`echo $table_path | rev | cut -d/ -f1 | rev` echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done # generate tables in subfolders @@ -54,30 +53,29 @@ done # * organfailure depends on measurement for d in demographics comorbidity measurement medication organfailure treatment firstday score sepsis; do - for fn in `ls $d`; + for fn_path in "$d"/*.sql; do - # only run SQL queries - if [[ "${fn: -4}" == ".sql" ]]; then - # table name is file name minus extension - tbl=`echo $fn | rev | cut -d. -f2- | rev` + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") + # table name is file name minus extension + tbl="${fn%.sql}" - # skip certain tables where order matters - skip=0 - for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 - do - if [[ "${tbl}" == "${skip_table}" ]]; then - skip=1 - break - fi - done; - if [[ "${skip}" == "1" ]]; then - continue - fi - - # not skipping - so generate the table on bigquery - echo "Generating ${TARGET_DATASET}.${tbl}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${tbl} < ${d}/${fn} + # skip certain tables where order matters + skip=0 + for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 + do + if [[ "${tbl}" == "${skip_table}" ]]; then + skip=1 + break + fi + done; + if [[ "${skip}" == "1" ]]; then + continue fi + + # not skipping - so generate the table on bigquery + echo "Generating ${TARGET_DATASET}.${tbl}" + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${tbl}" < "${fn_path}" done done @@ -88,5 +86,5 @@ do table=`echo $table_path | rev | cut -d/ -f1 | rev` echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done diff --git a/mimic-iv/concepts/validate_concepts.sh b/mimic-iv/concepts/validate_concepts.sh index eeaddb826..645a82a3e 100755 --- a/mimic-iv/concepts/validate_concepts.sh +++ b/mimic-iv/concepts/validate_concepts.sh @@ -32,7 +32,8 @@ fail=0 missing=0 while read -r tbl; do [ -z "${tbl}" ] && continue - if ! grep -q "^${tbl}," <<< "${ACTUAL}"; then + # exact field match (regex grep can false-hit prefixes) + if ! awk -F, -v t="${tbl}" '$1 == t { found=1 } END { exit !found }' <<< "${ACTUAL}"; then echo "MISSING: expected table ${DATASET}.${tbl} was not built" missing=1 fail=1 @@ -47,7 +48,7 @@ empty=0 while IFS=, read -r tbl rows; do [ -z "${tbl}" ] && continue # only validate the tables we actually build (ignore _metadata and any strays) - if grep -qx "${tbl}" <<< "${EXPECTED}" && [ "${rows}" -eq 0 ]; then + if grep -Fxq "${tbl}" <<< "${EXPECTED}" && [ "${rows:-}" -eq 0 ]; then echo "EMPTY: table ${DATASET}.${tbl} has 0 rows" empty=1 fail=1 diff --git a/src/mimic_utils/sqlglot_dialects/duckdb.py b/src/mimic_utils/sqlglot_dialects/duckdb.py index 12e83bfd0..2c3262b94 100644 --- a/src/mimic_utils/sqlglot_dialects/duckdb.py +++ b/src/mimic_utils/sqlglot_dialects/duckdb.py @@ -3,6 +3,10 @@ - ``NUMERIC`` is ``DECIMAL(38, 9)``, but DuckDB's bare ``DECIMAL`` defaults to ``DECIMAL(18, 3)``, which silently rounds values to three decimal places (e.g. ``CAST(0.0255 AS NUMERIC)`` becomes ``0.026`` before any explicit ``ROUND``). +- ``GENERATE_ARRAY`` must remain a LIST (for later ``UNNEST``). Native sqlglot +emits ``GENERATE_SERIES``, which is a set-returning function; wrapping as a +list matches Postgres ``ARRAY(GENERATE_SERIES)`` and keeps ``UNNEST(hrs)`` +reliable across DuckDB versions (#1736). """ import re @@ -41,6 +45,13 @@ def _parse_datetime_sql(self: DuckDB.Generator, expression: exp.ParseDatetime) - return f"STRPTIME({this}, {fmt})" +# GENERATE_ARRAY(a, b) -> list via generate_series (inclusive), for UNNEST. +def _generate_array_sql(self: DuckDB.Generator, expression: exp.Expression) -> str: + start = self.sql(expression, "start") + end = self.sql(expression, "end") + return f"(SELECT list(g) FROM generate_series({start}, {end}) AS t(g))" + + class MimicDuckDB(DuckDB): class Generator(DuckDB.Generator): def datatype_sql(self, expression: exp.DataType) -> str: @@ -54,4 +65,5 @@ def datatype_sql(self, expression: exp.DataType) -> str: **DuckDB.Generator.TRANSFORMS, exp.RegexpExtract: _regexp_extract_sql, exp.ParseDatetime: _parse_datetime_sql, + exp.GenerateSeries: _generate_array_sql, } diff --git a/src/mimic_utils/sqlglot_dialects/postgres.py b/src/mimic_utils/sqlglot_dialects/postgres.py index d33698558..509b369b2 100644 --- a/src/mimic_utils/sqlglot_dialects/postgres.py +++ b/src/mimic_utils/sqlglot_dialects/postgres.py @@ -23,8 +23,11 @@ def _unit(expression: exp.Expression, default: str = "DAY") -> str: # The logic is as follows: # * DAY -> difference of the two calendar dates (date subtraction = whole days) # * YEAR -> difference of the two calendar years +# * MONTH -> year*12 + month difference (calendar month boundaries) # * sub-day units -> truncate both operands to the unit (which makes the # elapsed seconds an exact multiple of the unit) then divide. +# WEEK is intentionally unsupported: BigQuery week starts (SUNDAY/ISO) do not +# match PostgreSQL DATE_TRUNC('week') Monday semantics. # https://cloud.google.com/bigquery/docs/reference/standard-sql/datetime_functions#datetime_diff _SECONDS_PER_UNIT = {"SECOND": 1, "MINUTE": 60, "HOUR": 3600} @@ -39,6 +42,19 @@ def _datetime_diff_sql(self: Postgres.Generator, expression: exp.Expression) -> return f"(CAST({end} AS DATE) - CAST({start} AS DATE))" if unit == "YEAR": return f"CAST(EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start}) AS BIGINT)" + if unit == "MONTH": + # Calendar month boundaries (matches BigQuery DATETIME_DIFF MONTH). + return ( + "CAST((EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start})) * 12 " + "+ (EXTRACT(MONTH FROM {end}) - EXTRACT(MONTH FROM {start})) AS BIGINT)" + ).format(end=end, start=start) + if unit not in _SECONDS_PER_UNIT: + # WEEK (and WEEK(SUNDAY)/ISO) need BigQuery's week-start semantics; + # refuse rather than emit a silently wrong days/7 division. + raise ValueError( + f"Unsupported DATETIME_DIFF unit {unit!r}; " + "expected SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR" + ) lo = unit.lower() factor = _SECONDS_PER_UNIT[unit] diff --git a/src/mimic_utils/transpile.py b/src/mimic_utils/transpile.py index d4a65541e..29924c187 100644 --- a/src/mimic_utils/transpile.py +++ b/src/mimic_utils/transpile.py @@ -113,7 +113,7 @@ def transpile_file( f"CREATE TABLE {derived_schema}{Path(source_file).stem} AS\n" ) + transpiled_query - with open(destination_file, "w") as write_file: + with open(destination_file, "w", encoding="utf-8") as write_file: write_file.write(transpiled_query) diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 952240bca..151d075f4 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -52,6 +52,9 @@ def t(bq: str, dialect: str) -> str: "SELECT (CAST(a.dischtime AS DATE) - CAST(a.admittime AS DATE)) FROM t AS a"), ("diff_year_pg", "SELECT DATETIME_DIFF(a.b, a.c, YEAR) FROM t a", "postgres", "SELECT CAST(EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c) AS BIGINT) FROM t AS a"), + ("diff_month_pg", "SELECT DATETIME_DIFF(a.b, a.c, MONTH) FROM t a", "postgres", + "SELECT CAST((EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c)) * 12 " + "+ (EXTRACT(MONTH FROM a.b) - EXTRACT(MONTH FROM a.c)) AS BIGINT) FROM t AS a"), # DuckDB handles DATETIME_DIFF natively (boundary count), operands swapped ("diff_hour_duckdb", "SELECT DATETIME_DIFF(a.outtime, a.intime, HOUR) FROM t a", "duckdb", "SELECT DATE_DIFF('HOUR', a.intime, a.outtime) FROM t AS a"), @@ -71,6 +74,9 @@ def t(bq: str, dialect: str) -> str: # GENERATE_ARRAY -> ARRAY(SELECT ... GENERATE_SERIES) (postgres) ("generate_array_pg", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "postgres", "SELECT ARRAY(SELECT * FROM GENERATE_SERIES(-24, 5)) AS hrs FROM t"), + # GENERATE_ARRAY -> list via generate_series (duckdb); must stay list for UNNEST + ("generate_array_duckdb", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "duckdb", + "SELECT (SELECT list(g) FROM generate_series(-24, 5) AS t(g)) AS hrs FROM t"), # handled natively by sqlglot 30.x (regression guards) ("datetime_date_pg", "SELECT DATETIME(me.chartdate) FROM t me", "postgres", @@ -216,6 +222,7 @@ def test_mimic_iii_concept_transpiles_and_reparses(sql_file, dialect): ("2150-01-02 01:00:00", "2150-01-01 23:00:00", "DAY", 1), # crosses midnight ("2150-01-01 23:00:00", "2150-01-01 01:00:00", "DAY", 0), # same day ("2155-06-15 00:00:00", "2150-01-01 00:00:00", "YEAR", 5), + ("2150-04-01 00:00:00", "2150-01-15 00:00:00", "MONTH", 3), ("2150-01-01 00:01:00", "2150-01-01 00:00:59", "MINUTE", 1), ("2150-01-01 00:00:30", "2150-01-01 00:00:10", "SECOND", 20), ("2150-01-01 01:00:00", "2150-01-01 03:00:00", "HOUR", -2), # negative direction