diff --git a/.github/workflows/lint_sqlfluff.yml b/.github/workflows/lint_sqlfluff.yml index 9401198e..ca4c8c8b 100644 --- a/.github/workflows/lint_sqlfluff.yml +++ b/.github/workflows/lint_sqlfluff.yml @@ -35,8 +35,12 @@ jobs: shell: bash run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json - name: Annotate + # Fork PRs cannot create check runs with GITHUB_TOKEN; lint already ran. + continue-on-error: true + if: steps.get_files_to_lint.outputs.lintees != '' uses: yuzutech/annotations-action@v0.6.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" title: "SQLFluff Lint" - input: "./annotations.json" \ No newline at end of file + input: "./annotations.json" + ignore-missing-file: true \ No newline at end of file diff --git a/mimic-iii/buildmimic/duckdb/import_duckdb.sh b/mimic-iii/buildmimic/duckdb/import_duckdb.sh index 77b76c7d..badf83b8 100755 --- a/mimic-iii/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iii/buildmimic/duckdb/import_duckdb.sh @@ -29,7 +29,7 @@ usage () { die " USAGE: ./import_duckdb.sh mimic_data_dir [output_db] WHERE: - mimic_data_dir directory that contains csv.gz or csv files + mimic_data_dir directory that contains csv.tar.gz or csv files output_db: optional filename for duckdb file (default: mimic3.db)\ " } @@ -84,9 +84,11 @@ make_table_name () { # load data into database find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do make_table_name "$FILE" - echo "Loading $FILE .. \c" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") + printf "Loading %s .. " "$FILE" try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL echo "done!" done && echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iii/buildmimic/postgres/Makefile b/mimic-iii/buildmimic/postgres/Makefile index 15b4c521..29953adb 100644 --- a/mimic-iii/buildmimic/postgres/Makefile +++ b/mimic-iii/buildmimic/postgres/Makefile @@ -104,7 +104,7 @@ mimic-build-gz: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v mimic_data_dir=${datadir} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -151,7 +151,7 @@ mimic-build: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v mimic_data_dir=${DATADIR} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -184,7 +184,7 @@ ifeq ("$(physionetuser)","") @echo 'Call the makefile again with physionetuser=' @echo ' e.g. make eicu-download datadir=/path/to/data physionetuser=hello@physionet.org' else - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" endif mimic-demo-download: @@ -192,7 +192,7 @@ mimic-demo-download: @echo '-- Downloading MIMIC-III from PhysioNet --' @echo '------------------------------------------' @echo '' - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" #This is fairly inelegant and could be tidied with a for loop and an if to check for gzip, #but need to maintain compatibility with Windows, which baffling lacks these things @@ -281,7 +281,7 @@ concepts: @echo '---------------------' @echo '' @sleep 2 - cd ../../concepts_postgres/ && psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -c "CREATE SCHEMA IF NOT EXISTS mimiciii_derived;" -f postgres-make-concepts.sql + cd ../../concepts/ && psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f make-concepts.sql .PHONY: help mimic clean diff --git a/mimic-iii/buildmimic/postgres/create_mimic_user.sh b/mimic-iii/buildmimic/postgres/create_mimic_user.sh index 772764ce..d639f5fc 100755 --- a/mimic-iii/buildmimic/postgres/create_mimic_user.sh +++ b/mimic-iii/buildmimic/postgres/create_mimic_user.sh @@ -20,6 +20,13 @@ else echo "User is set to '$MIMIC_USER'"; fi +# escape ' in password for SQL string literal +MIMIC_PASSWORD_SQL=$(printf '%s' "$MIMIC_PASSWORD" | sed "s/'/''/g") +# quote SQL identifiers (double any embedded ") +sql_ident () { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; } +MIMIC_USER_SQL=$(sql_ident "$MIMIC_USER") +MIMIC_DB_SQL=$(sql_ident "$MIMIC_DB") + PSQL='psql' # add in the host/port, if they were specified (not null, -n) @@ -58,11 +65,11 @@ fi if [ "$MIMIC_USER" != "postgres" ]; then # we need to create this user via postgres # use SUDO to login as postgres - $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER; CREATE USER $MIMIC_USER WITH PASSWORD '$MIMIC_PASSWORD';" + $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER_SQL; CREATE USER $MIMIC_USER_SQL WITH PASSWORD '$MIMIC_PASSWORD_SQL';" fi if [ "$MIMIC_DB" != "postgres" ]; then # drop and recreate the database - $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB;" - $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB OWNER $MIMIC_USER;" -fi \ No newline at end of file + $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB_SQL;" + $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB_SQL OWNER $MIMIC_USER_SQL;" +fi diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index 801651c7..4e0113d3 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -10,6 +10,34 @@ CHUNKSIZE = 10 ** 6 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME) +# Column dtypes for tables that trigger pandas mixed-type warnings when +# loaded with the default low_memory chunked inference (see #1237). +# Types mirror mimic-iii/buildmimic/postgres/postgres_create_tables.sql. +TABLE_DTYPES = { + "chartevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "datetimeevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "inputevents_cv": { + "ORIGINALRATE": "float64", + "ORIGINALRATEUOM": "string", + "ORIGINALSITE": "string", + }, + "noteevents": { + "CHARTTIME": "string", + "STORETIME": "string", + "ISERROR": "string", + }, +} + def _table_name_from_csv(filename: str) -> str: """Derive SQL table name from a CSV path (literal suffix, not str.strip).""" @@ -21,20 +49,39 @@ def _table_name_from_csv(filename: str) -> str: return name.lower() +def _read_csv(path, table, **kwargs): + # low_memory=False avoids dtype re-inference across chunks; table-specific + # dtypes pin the columns that otherwise flip between numeric/object. + return pd.read_csv( + path, + index_col="ROW_ID", + low_memory=False, + dtype=TABLE_DTYPES.get(table), + **kwargs, + ) + + if os.path.exists(DATABASE_NAME): msg = "File {} already exists.".format(DATABASE_NAME) print(msg) sys.exit() +# Prefer .csv.gz when both exist for the same table; also load plain .csv +# (import.sh already accepts both). +files_by_table = {} +for f in glob("*.csv"): + files_by_table[_table_name_from_csv(f)] = f for f in glob("*.csv.gz"): + files_by_table[_table_name_from_csv(f)] = f + +for table, f in sorted(files_by_table.items()): print("Starting processing {}".format(f)) - table = _table_name_from_csv(f) if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, index_col="ROW_ID") + df = _read_csv(f, table) df.to_sql(table, CONNECTION_STRING) else: # If the file is too large, let's do the work in chunks - for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE): + for chunk in _read_csv(f, table, chunksize=CHUNKSIZE): chunk.to_sql(table, CONNECTION_STRING, if_exists="append") print("Finished processing {}".format(f)) diff --git a/mimic-iii/buildmimic/sqlite/import.sh b/mimic-iii/buildmimic/sqlite/import.sh index dbf6b9ce..58b85abb 100755 --- a/mimic-iii/buildmimic/sqlite/import.sh +++ b/mimic-iii/buildmimic/sqlite/import.sh @@ -28,11 +28,11 @@ for FILE in *; do TABLE_NAME=$(echo "${FILE%%.*}" | tr "[:upper:]" "[:lower:]") case "$FILE" in *csv) - IMPORT_CMD=".import $FILE $TABLE_NAME" + IMPORT_CMD=".import \"$FILE\" $TABLE_NAME" ;; # need to decompress csv before load *csv.gz) - IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME" + IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME" ;; # not a data file so skip *) @@ -40,7 +40,7 @@ for FILE in *; do ;; esac echo "Loading $FILE." - sqlite3 $OUTFILE < "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-ed # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (ed) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .. " try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected ed/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh index b194212f..487d7405 100644 --- a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh @@ -29,7 +29,7 @@ usage () { die " USAGE: ./import_duckdb.sh mimic_data_dir [output_db] WHERE: - mimic_data_dir directory that contains csv.gz or csv files + mimic_data_dir directory that contains csv.tar.gz or csv files output_db: optional filename for duckdb file (default: mimic4_note.db)\ " } @@ -96,18 +96,23 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-note # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (note) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .." OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -122,5 +127,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected note/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index 8e72733c..32f608c4 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -29,7 +29,7 @@ usage () { die " USAGE: ./import_duckdb.sh mimic_data_dir [output_db] WHERE: - mimic_data_dir directory that contains csv.gz or csv files + mimic_data_dir directory that contains csv.tar.gz or csv files output_db: optional filename for duckdb file (default: mimic4.db)\ " } @@ -101,18 +101,26 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both uncompressed (.csv) and gzip (.csv.gz). The old '*.csv???' +# pattern only matched names with exactly three chars after ".csv" (i.e. .gz), +# so plain .csv files were silently skipped while the script still reported success. +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv # avoids syntax errors if mimic-iv-ed in the same dir - case $DIRNAME in + case "$DIRNAME" in (hosp|icu) ;; # OK (*) continue; esac - echo "Loading $FILE .. \c" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") + printf "Loading %s .. " "$FILE" OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -127,5 +135,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected hosp/ and icu/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/mysql/validate_demo.sql b/mimic-iv/buildmimic/mysql/validate_demo.sql index 575e8413..ebf4f02d 100644 --- a/mimic-iv/buildmimic/mysql/validate_demo.sql +++ b/mimic-iv/buildmimic/mysql/validate_demo.sql @@ -31,14 +31,17 @@ FROM ( SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) exp @@ -64,14 +67,17 @@ INNER JOIN SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM procedureevents ) obs diff --git a/mimic-iv/buildmimic/postgres/validate_demo.sql b/mimic-iv/buildmimic/postgres/validate_demo.sql index 303dd98e..049497e5 100644 --- a/mimic-iv/buildmimic/postgres/validate_demo.sql +++ b/mimic-iv/buildmimic/postgres/validate_demo.sql @@ -22,14 +22,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) @@ -54,14 +57,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM mimiciv_hosp.poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM mimiciv_hosp.prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM mimiciv_hosp.procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM mimiciv_hosp.provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM mimiciv_hosp.services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM mimiciv_hosp.transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM mimiciv_icu.icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM mimiciv_icu.caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM mimiciv_icu.d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.procedureevents ) diff --git a/mimic-iv/buildmimic/sqlite/import.py b/mimic-iv/buildmimic/sqlite/import.py index 68d43647..31b880c4 100644 --- a/mimic-iv/buildmimic/sqlite/import.py +++ b/mimic-iv/buildmimic/sqlite/import.py @@ -110,7 +110,7 @@ def main(): if args.limit > 0: for f in data_files: if 'patients' in f.name: - pt = pd.read_csv(f) + pt = pd.read_csv(f, low_memory=False) break if pt is None: raise FileNotFoundError('Unable to find a patients file in current folder.') @@ -158,7 +158,7 @@ def main(): tablename = tablenames[i] print("Starting processing {}".format(tablename), end='.. ') if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, dtype=mimic_dtypes) + df = pd.read_csv(f, dtype=mimic_dtypes, low_memory=False) df = process_dataframe(df, subjects=subjects) df.to_sql(tablename, connection, index=False) row_counts[tablename] += len(df) diff --git a/mimic-iv/buildmimic/sqlite/import.sh b/mimic-iv/buildmimic/sqlite/import.sh index 08ded288..fb642d1c 100755 --- a/mimic-iv/buildmimic/sqlite/import.sh +++ b/mimic-iv/buildmimic/sqlite/import.sh @@ -29,11 +29,11 @@ for FILE in */**.csv*; do TABLE_NAME=$(echo "${BASENAME%%.*}" | tr "[:upper:]" "[:lower:]") case "$FILE" in *csv) - IMPORT_CMD=".import $FILE $TABLE_NAME" + IMPORT_CMD=".import \"$FILE\" $TABLE_NAME" ;; # need to decompress csv before load *csv.gz) - IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME" + IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME" ;; # not a data file so skip *) @@ -41,7 +41,7 @@ for FILE in */**.csv*; do ;; esac echo "Loading $FILE." - sqlite3 $OUTFILE <" exit 1 fi diff --git a/mimic-iv/concepts/make_concepts.sh b/mimic-iv/concepts/make_concepts.sh index 112bcdc2..42be9d22 100644 --- a/mimic-iv/concepts/make_concepts.sh +++ b/mimic-iv/concepts/make_concepts.sh @@ -8,16 +8,15 @@ export MIMIC_VERSION="3.1" # note: max_rows=1 *displays* only one row, but all rows are inserted into the destination table BQ_OPTIONS='--quiet --headless --max_rows=0 --use_legacy_sql=False --replace' -# drop the existing tables in the target dataset -for TABLE in `bq ls physionet-data:${TARGET_DATASET} | cut -d' ' -f3`; -do - # skip the first line of dashes - if [[ "${TABLE:0:2}" == '--' ]]; then - continue - fi +# drop existing tables (CSV format: cut -f3 on pretty ls mis-parses names) +while IFS= read -r TABLE; do + [ -z "${TABLE}" ] && continue echo "Dropping table ${TARGET_DATASET}.${TABLE}" - bq rm -f -q ${TARGET_DATASET}.${TABLE} -done + bq rm -f -q "${TARGET_DATASET}.${TABLE}" +done < <( + bq ls --format=csv --max_results=10000 "physionet-data:${TARGET_DATASET}" \ + | awk -F, 'NR > 1 { print $1 }' +) # create a _version table to store the mimic-iv version, git commit hash, and latest git tag GIT_COMMIT_HASH=$(git rev-parse HEAD) @@ -44,7 +43,7 @@ for table_path in demographics/icustay_times; do table=`echo $table_path | rev | cut -d/ -f1 | rev` echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done # generate tables in subfolders @@ -54,30 +53,29 @@ done # * organfailure depends on measurement for d in demographics comorbidity measurement medication organfailure treatment firstday score sepsis; do - for fn in `ls $d`; + for fn_path in "$d"/*.sql; do - # only run SQL queries - if [[ "${fn: -4}" == ".sql" ]]; then - # table name is file name minus extension - tbl=`echo $fn | rev | cut -d. -f2- | rev` + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") + # table name is file name minus extension + tbl="${fn%.sql}" - # skip certain tables where order matters - skip=0 - for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 - do - if [[ "${tbl}" == "${skip_table}" ]]; then - skip=1 - break - fi - done; - if [[ "${skip}" == "1" ]]; then - continue - fi - - # not skipping - so generate the table on bigquery - echo "Generating ${TARGET_DATASET}.${tbl}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${tbl} < ${d}/${fn} + # skip certain tables where order matters + skip=0 + for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 + do + if [[ "${tbl}" == "${skip_table}" ]]; then + skip=1 + break + fi + done; + if [[ "${skip}" == "1" ]]; then + continue fi + + # not skipping - so generate the table on bigquery + echo "Generating ${TARGET_DATASET}.${tbl}" + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${tbl}" < "${fn_path}" done done @@ -88,5 +86,5 @@ do table=`echo $table_path | rev | cut -d/ -f1 | rev` echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done diff --git a/mimic-iv/concepts/validate_concepts.sh b/mimic-iv/concepts/validate_concepts.sh index eeaddb82..645a82a3 100755 --- a/mimic-iv/concepts/validate_concepts.sh +++ b/mimic-iv/concepts/validate_concepts.sh @@ -32,7 +32,8 @@ fail=0 missing=0 while read -r tbl; do [ -z "${tbl}" ] && continue - if ! grep -q "^${tbl}," <<< "${ACTUAL}"; then + # exact field match (regex grep can false-hit prefixes) + if ! awk -F, -v t="${tbl}" '$1 == t { found=1 } END { exit !found }' <<< "${ACTUAL}"; then echo "MISSING: expected table ${DATASET}.${tbl} was not built" missing=1 fail=1 @@ -47,7 +48,7 @@ empty=0 while IFS=, read -r tbl rows; do [ -z "${tbl}" ] && continue # only validate the tables we actually build (ignore _metadata and any strays) - if grep -qx "${tbl}" <<< "${EXPECTED}" && [ "${rows}" -eq 0 ]; then + if grep -Fxq "${tbl}" <<< "${EXPECTED}" && [ "${rows:-}" -eq 0 ]; then echo "EMPTY: table ${DATASET}.${tbl} has 0 rows" empty=1 fail=1 diff --git a/src/mimic_utils/sqlglot_dialects/duckdb.py b/src/mimic_utils/sqlglot_dialects/duckdb.py index 12e83bfd..2c3262b9 100644 --- a/src/mimic_utils/sqlglot_dialects/duckdb.py +++ b/src/mimic_utils/sqlglot_dialects/duckdb.py @@ -3,6 +3,10 @@ - ``NUMERIC`` is ``DECIMAL(38, 9)``, but DuckDB's bare ``DECIMAL`` defaults to ``DECIMAL(18, 3)``, which silently rounds values to three decimal places (e.g. ``CAST(0.0255 AS NUMERIC)`` becomes ``0.026`` before any explicit ``ROUND``). +- ``GENERATE_ARRAY`` must remain a LIST (for later ``UNNEST``). Native sqlglot +emits ``GENERATE_SERIES``, which is a set-returning function; wrapping as a +list matches Postgres ``ARRAY(GENERATE_SERIES)`` and keeps ``UNNEST(hrs)`` +reliable across DuckDB versions (#1736). """ import re @@ -41,6 +45,13 @@ def _parse_datetime_sql(self: DuckDB.Generator, expression: exp.ParseDatetime) - return f"STRPTIME({this}, {fmt})" +# GENERATE_ARRAY(a, b) -> list via generate_series (inclusive), for UNNEST. +def _generate_array_sql(self: DuckDB.Generator, expression: exp.Expression) -> str: + start = self.sql(expression, "start") + end = self.sql(expression, "end") + return f"(SELECT list(g) FROM generate_series({start}, {end}) AS t(g))" + + class MimicDuckDB(DuckDB): class Generator(DuckDB.Generator): def datatype_sql(self, expression: exp.DataType) -> str: @@ -54,4 +65,5 @@ def datatype_sql(self, expression: exp.DataType) -> str: **DuckDB.Generator.TRANSFORMS, exp.RegexpExtract: _regexp_extract_sql, exp.ParseDatetime: _parse_datetime_sql, + exp.GenerateSeries: _generate_array_sql, } diff --git a/src/mimic_utils/sqlglot_dialects/postgres.py b/src/mimic_utils/sqlglot_dialects/postgres.py index d3369855..509b369b 100644 --- a/src/mimic_utils/sqlglot_dialects/postgres.py +++ b/src/mimic_utils/sqlglot_dialects/postgres.py @@ -23,8 +23,11 @@ def _unit(expression: exp.Expression, default: str = "DAY") -> str: # The logic is as follows: # * DAY -> difference of the two calendar dates (date subtraction = whole days) # * YEAR -> difference of the two calendar years +# * MONTH -> year*12 + month difference (calendar month boundaries) # * sub-day units -> truncate both operands to the unit (which makes the # elapsed seconds an exact multiple of the unit) then divide. +# WEEK is intentionally unsupported: BigQuery week starts (SUNDAY/ISO) do not +# match PostgreSQL DATE_TRUNC('week') Monday semantics. # https://cloud.google.com/bigquery/docs/reference/standard-sql/datetime_functions#datetime_diff _SECONDS_PER_UNIT = {"SECOND": 1, "MINUTE": 60, "HOUR": 3600} @@ -39,6 +42,19 @@ def _datetime_diff_sql(self: Postgres.Generator, expression: exp.Expression) -> return f"(CAST({end} AS DATE) - CAST({start} AS DATE))" if unit == "YEAR": return f"CAST(EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start}) AS BIGINT)" + if unit == "MONTH": + # Calendar month boundaries (matches BigQuery DATETIME_DIFF MONTH). + return ( + "CAST((EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start})) * 12 " + "+ (EXTRACT(MONTH FROM {end}) - EXTRACT(MONTH FROM {start})) AS BIGINT)" + ).format(end=end, start=start) + if unit not in _SECONDS_PER_UNIT: + # WEEK (and WEEK(SUNDAY)/ISO) need BigQuery's week-start semantics; + # refuse rather than emit a silently wrong days/7 division. + raise ValueError( + f"Unsupported DATETIME_DIFF unit {unit!r}; " + "expected SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR" + ) lo = unit.lower() factor = _SECONDS_PER_UNIT[unit] diff --git a/src/mimic_utils/transpile.py b/src/mimic_utils/transpile.py index d4a65541..29924c18 100644 --- a/src/mimic_utils/transpile.py +++ b/src/mimic_utils/transpile.py @@ -113,7 +113,7 @@ def transpile_file( f"CREATE TABLE {derived_schema}{Path(source_file).stem} AS\n" ) + transpiled_query - with open(destination_file, "w") as write_file: + with open(destination_file, "w", encoding="utf-8") as write_file: write_file.write(transpiled_query) diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 952240bc..151d075f 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -52,6 +52,9 @@ def t(bq: str, dialect: str) -> str: "SELECT (CAST(a.dischtime AS DATE) - CAST(a.admittime AS DATE)) FROM t AS a"), ("diff_year_pg", "SELECT DATETIME_DIFF(a.b, a.c, YEAR) FROM t a", "postgres", "SELECT CAST(EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c) AS BIGINT) FROM t AS a"), + ("diff_month_pg", "SELECT DATETIME_DIFF(a.b, a.c, MONTH) FROM t a", "postgres", + "SELECT CAST((EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c)) * 12 " + "+ (EXTRACT(MONTH FROM a.b) - EXTRACT(MONTH FROM a.c)) AS BIGINT) FROM t AS a"), # DuckDB handles DATETIME_DIFF natively (boundary count), operands swapped ("diff_hour_duckdb", "SELECT DATETIME_DIFF(a.outtime, a.intime, HOUR) FROM t a", "duckdb", "SELECT DATE_DIFF('HOUR', a.intime, a.outtime) FROM t AS a"), @@ -71,6 +74,9 @@ def t(bq: str, dialect: str) -> str: # GENERATE_ARRAY -> ARRAY(SELECT ... GENERATE_SERIES) (postgres) ("generate_array_pg", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "postgres", "SELECT ARRAY(SELECT * FROM GENERATE_SERIES(-24, 5)) AS hrs FROM t"), + # GENERATE_ARRAY -> list via generate_series (duckdb); must stay list for UNNEST + ("generate_array_duckdb", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "duckdb", + "SELECT (SELECT list(g) FROM generate_series(-24, 5) AS t(g)) AS hrs FROM t"), # handled natively by sqlglot 30.x (regression guards) ("datetime_date_pg", "SELECT DATETIME(me.chartdate) FROM t me", "postgres", @@ -216,6 +222,7 @@ def test_mimic_iii_concept_transpiles_and_reparses(sql_file, dialect): ("2150-01-02 01:00:00", "2150-01-01 23:00:00", "DAY", 1), # crosses midnight ("2150-01-01 23:00:00", "2150-01-01 01:00:00", "DAY", 0), # same day ("2155-06-15 00:00:00", "2150-01-01 00:00:00", "YEAR", 5), + ("2150-04-01 00:00:00", "2150-01-15 00:00:00", "MONTH", 3), ("2150-01-01 00:01:00", "2150-01-01 00:00:59", "MINUTE", 1), ("2150-01-01 00:00:30", "2150-01-01 00:00:10", "SECOND", 20), ("2150-01-01 01:00:00", "2150-01-01 03:00:00", "HOUR", -2), # negative direction