fix: load paths, imports, transpile, and CI permissions

.github/workflows/lint_sqlfluff.yml

@@ -35,8 +35,12 @@ jobs:
         shell: bash
         run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json
       - name: Annotate
+        # Fork PRs cannot create check runs with GITHUB_TOKEN; lint already ran.
+        continue-on-error: true
+        if: steps.get_files_to_lint.outputs.lintees != ''
         uses: yuzutech/annotations-action@v0.6.0
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           title: "SQLFluff Lint"
-          input: "./annotations.json"
+          input: "./annotations.json"
+          ignore-missing-file: true

mimic-iii/buildmimic/duckdb/import_duckdb.sh

@@ -84,9 +84,11 @@ make_table_name () {
 # load data into database
 find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do
     make_table_name "$FILE"
-    echo "Loading $FILE .. \c"
+    # Escape single quotes for SQL string literal
+    FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g")
+    printf "Loading %s .. " "$FILE"
     try duckdb "$OUTFILE" <<-EOSQL
-		COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
+		COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
 EOSQL
     echo "done!"
 done && echo "Successfully finished loading data into $OUTFILE."

mimic-iii/buildmimic/postgres/Makefile

@@ -104,7 +104,7 @@ mimic-build-gz:
 	@echo '------------------'
 	@echo ''
 	@sleep 2
-	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v mimic_data_dir=${datadir}
+	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v "mimic_data_dir=$(DATADIR)"
 	@echo ''
 	@echo '--------------------'
 	@echo '-- Adding indexes --'
@@ -151,7 +151,7 @@ mimic-build:
 	@echo '------------------'
 	@echo ''
 	@sleep 2
-	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v mimic_data_dir=${DATADIR}
+	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v "mimic_data_dir=$(DATADIR)"
 	@echo ''
 	@echo '--------------------'
 	@echo '-- Adding indexes --'
@@ -184,15 +184,15 @@ ifeq ("$(physionetuser)","")
 	@echo 'Call the makefile again with physionetuser=<USERNAME>'
 	@echo ' e.g. make eicu-download datadir=/path/to/data physionetuser=hello@physionet.org'
 else
-	wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)"
+	wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)"
 endif
 
 mimic-demo-download:
 	@echo '------------------------------------------'
 	@echo '-- Downloading MIMIC-III from PhysioNet --'
 	@echo '------------------------------------------'
 	@echo ''
-	wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)"
+	wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)"
 
 #This is fairly inelegant and could be tidied with a for loop and an if to check for gzip,
 #but need to maintain compatibility with Windows, which baffling lacks these things

mimic-iii/buildmimic/postgres/create_mimic_user.sh

@@ -20,6 +20,13 @@ else
   echo "User is set to '$MIMIC_USER'";
 fi
 
+# escape ' in password for SQL string literal
+MIMIC_PASSWORD_SQL=$(printf '%s' "$MIMIC_PASSWORD" | sed "s/'/''/g")
+# quote SQL identifiers (double any embedded ")
+sql_ident () { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; }
+MIMIC_USER_SQL=$(sql_ident "$MIMIC_USER")
+MIMIC_DB_SQL=$(sql_ident "$MIMIC_DB")
+
 PSQL='psql'
 
 # add in the host/port, if they were specified (not null, -n)
@@ -58,11 +65,11 @@ fi
 if [ "$MIMIC_USER" != "postgres" ]; then
     # we need to create this user via postgres
     # use SUDO to login as postgres
-    $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER; CREATE USER $MIMIC_USER WITH PASSWORD '$MIMIC_PASSWORD';"
+    $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER_SQL; CREATE USER $MIMIC_USER_SQL WITH PASSWORD '$MIMIC_PASSWORD_SQL';"
 fi
 
 if [ "$MIMIC_DB" != "postgres" ]; then
   # drop and recreate the database
-  $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB;"
-  $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB OWNER $MIMIC_USER;"
-fi
+  $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB_SQL;"
+  $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB_SQL OWNER $MIMIC_USER_SQL;"
+fi

mimic-iii/buildmimic/sqlite/import.py

@@ -10,6 +10,34 @@
 CHUNKSIZE = 10 ** 6
 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME)
 
+# Column dtypes for tables that trigger pandas mixed-type warnings when
+# loaded with the default low_memory chunked inference (see #1237).
+# Types mirror mimic-iii/buildmimic/postgres/postgres_create_tables.sql.
+TABLE_DTYPES = {
+    "chartevents": {
+        "WARNING": "Int64",
+        "ERROR": "Int64",
+        "RESULTSTATUS": "string",
+        "STOPPED": "string",
+    },
+    "datetimeevents": {
+        "WARNING": "Int64",
+        "ERROR": "Int64",
+        "RESULTSTATUS": "string",
+        "STOPPED": "string",
+    },
+    "inputevents_cv": {
+        "ORIGINALRATE": "float64",
+        "ORIGINALRATEUOM": "string",
+        "ORIGINALSITE": "string",
+    },
+    "noteevents": {
+        "CHARTTIME": "string",
+        "STORETIME": "string",
+        "ISERROR": "string",
+    },
+}
+
 
 def _table_name_from_csv(filename: str) -> str:
     """Derive SQL table name from a CSV path (literal suffix, not str.strip)."""
@@ -21,20 +49,39 @@ def _table_name_from_csv(filename: str) -> str:
     return name.lower()
 
 
+def _read_csv(path, table, **kwargs):
+    # low_memory=False avoids dtype re-inference across chunks; table-specific
+    # dtypes pin the columns that otherwise flip between numeric/object.
+    return pd.read_csv(
+        path,
+        index_col="ROW_ID",
+        low_memory=False,
+        dtype=TABLE_DTYPES.get(table),
+        **kwargs,
+    )
+
+
 if os.path.exists(DATABASE_NAME):
     msg = "File {} already exists.".format(DATABASE_NAME)
     print(msg)
     sys.exit()
 
+# Prefer .csv.gz when both exist for the same table; also load plain .csv
+# (import.sh already accepts both).
+files_by_table = {}
+for f in glob("*.csv"):
+    files_by_table[_table_name_from_csv(f)] = f
 for f in glob("*.csv.gz"):
+    files_by_table[_table_name_from_csv(f)] = f
+
+for table, f in sorted(files_by_table.items()):
     print("Starting processing {}".format(f))
-    table = _table_name_from_csv(f)
     if os.path.getsize(f) < THRESHOLD_SIZE:
-        df = pd.read_csv(f, index_col="ROW_ID")
+        df = _read_csv(f, table)
         df.to_sql(table, CONNECTION_STRING)
     else:
         # If the file is too large, let's do the work in chunks
-        for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE):
+        for chunk in _read_csv(f, table, chunksize=CHUNKSIZE):
             chunk.to_sql(table, CONNECTION_STRING, if_exists="append")
     print("Finished processing {}".format(f))
 

mimic-iii/buildmimic/sqlite/import.sh

@@ -28,19 +28,19 @@ for FILE in *; do
     TABLE_NAME=$(echo "${FILE%%.*}" | tr "[:upper:]" "[:lower:]")
     case "$FILE" in
         *csv)
-            IMPORT_CMD=".import $FILE $TABLE_NAME"
+            IMPORT_CMD=".import \"$FILE\" $TABLE_NAME"
         ;;
         # need to decompress csv before load
         *csv.gz)
-            IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME"
+            IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME"
         ;;
         # not a data file so skip
         *)
             continue
         ;;
     esac
     echo "Loading $FILE."
-    sqlite3 $OUTFILE <<EOF
+    sqlite3 "$OUTFILE" <<EOF
 .headers on
 .mode csv
 $IMPORT_CMD

mimic-iii/concepts/README.md

@@ -5,7 +5,7 @@ The scripts are intended to be run against the BigQuery instantiation of MIMIC-I
 Concepts are categorized into folders if possible, otherwise they remain in the top-level directory.
 A table of contents is provided below: [List of concepts](#list-of-concepts).
 
-You can read about cloud access to MIMIC-III, including via Google BigQuery, on the [cloud page](https://mimic.physionet.org/gettingstarted/cloud/).
+You can read about cloud access to MIMIC-III, including via Google BigQuery, on the [cloud page](https://mimic.mit.edu/docs/gettingstarted/cloud/).
 
 The rest of this README describes:
 
@@ -14,7 +14,7 @@ The rest of this README describes:
 
 ## Generating the concepts in BigQuery
 
-You do not need to generate the concepts if you are using BigQuery! They have already been generated for you. If you have access to MIMIC-III on BigQuery, look under `physionet-data.mimic_derived`. If you would like to generate the concepts again, for example on your own dataset, you must modify the `TARGET_DATASET` variable within the [make-concepts.sh](/concepts/make-concepts.sh) script. The script assumes you have installed and configured the [Google Cloud SDK](https://cloud.google.com/sdk/docs/install).
+You do not need to generate the concepts if you are using BigQuery! They have already been generated for you. If you have access to MIMIC-III on BigQuery, look under `physionet-data.mimiciii_derived`. If you would like to generate the concepts again, for example on your own dataset, you must modify the `TARGET_DATASET` variable within the [make-concepts.sh](/mimic-iii/concepts/make-concepts.sh) script. The script assumes you have installed and configured the [Google Cloud SDK](https://cloud.google.com/sdk/docs/install).
 
 ## Generating the concepts in PostgreSQL or DuckDB
 
@@ -42,6 +42,7 @@ Folder | Table | Description
 --- | --- | ---
 . | [echo_data](echo_data.sql) | Text extracted from echocardiography reports using regular expressions.
 . | [code_status](code_status.sql) | Whether the patient has restrictions on life saving resuscitation.
+. | [rrt](rrt.sql) | Whether the patient received any dialysis during their ICU stay.
 comorbidity | [elixhauser_ahrq_v37](comorbidity/elixhauser_ahrq_v37.sql)                    | Comorbidities in categories proposed by Elixhauser et al. AHRQ produced the mapping.
 comorbidity | [elixhauser_ahrq_v37_no_drg](comorbidity/elixhauser_ahrq_v37_no_drg.sql)      | As above, but DRG codes are not used to exclude primary conditions.
 comorbidity | [elixhauser_quan](comorbidity/elixhauser_quan.sql)                            | Comorbidities in categories proposed by Elixhauser et al. using an algorithm by Quan et al.
@@ -76,7 +77,7 @@ durations | [neuroblock_dose](durations/neuroblock_dose.sql)
 durations | [norepinephrine_dose](durations/norepinephrine_dose.sql)                        | Dose administered with start/stop times for norepinephrine.
 durations | [phenylephrine_dose](durations/phenylephrine_dose.sql)                          | Dose administered with start/stop times for phenylephrine.
 durations | [vasopressin_dose](durations/vasopressin_dose.sql)                              | Dose administered with start/stop times for vasopressin.
-durations | [ventilation_classification](durations/ventilation_classification.sql)          | Classifies patient settings as implying mechanical ventilation.
+durations | [ventilation_classification](durations/ventilation_classification.sql)          | Classifies ventilator settings (tidal volume, PEEP, etc.) as implying mechanical ventilation.
 durations | [ventilation_durations](durations/ventilation_durations.sql)                    | Start and stop times for mechanical ventilation.
 durations | [weight_durations](durations/weight_durations.sql)                              | Start and stop times for daily weight measurements.
 **firstday** | | The first day subfolder contains scripts to summarizes a patient's health on their first ICU day.
@@ -93,6 +94,8 @@ firstday | [weight_first_day](firstday/weight_first_day.sql)
 **fluid_balance** | | Tables which track fluid input and output for the patient.
 fluid_balance | [colloid_bolus](fluid_balance/colloid_bolus.sql)                            | Times at which a patient received a bolus of colloidal fluid.
 fluid_balance | [crystalloid_bolus](fluid_balance/crystalloid_bolus.sql)                    | Times at which a patient received a bolus of crystalloid fluid.
+fluid_balance | [ffp_transfusion](fluid_balance/ffp_transfusion.sql)                        | Times at which a patient received a transfusion of fresh frozen plasma.
+fluid_balance | [rbc_transfusion](fluid_balance/rbc_transfusion.sql)                        | Times at which a patient received a transfusion of red blood cells.
 fluid_balance | [urine_output](fluid_balance/urine_output.sql)                              | Urine output for a patient with the time of documentation.
 **organfailure** | | Summarizations of the degree of organ failure for single organ systems.
 organfailure | [kdigo_creatinine](organfailure/kdigo_creatinine.sql)                        | Creatinine values with baseline creatinine as defined by KDIGO.
@@ -103,6 +106,7 @@ organfailure | [kdigo_stages_7day](organfailure/kdigo_stages_7day.sql)
 organfailure | [meld](organfailure/meld.sql)                                                | The MELD score, often used to assess health of liver transplant candidates.
 **pivot** |                                                                                 | Pivoted views contain the patient `icustay_id`, the `charttime`, and a number of variables. They are useful to acquiring a time series of values for patient stays.
 pivot | [pivoted_bg](pivot/pivoted_bg.sql)                                                  | Blood gas measurements.
+pivot | [pivoted_bg_art](pivot/pivoted_bg_art.sql)                                          | Arterial blood gas measurements, explicitly labelled or inferred from SpO2.
 pivot | [pivoted_fio2](pivot/pivoted_fio2.sql)                                              | Fraction of inspired oxygen.
 pivot | [pivoted_gcs](pivot/pivoted_gcs.sql)                                                | Glasgow Coma Scale.
 pivot | [pivoted_height](pivot/pivoted_height.sql)                                          | Height.
@@ -113,7 +117,6 @@ pivot | [pivoted_oasis](pivot/pivoted_oasis.sql)
 pivot | [pivoted_rrt](pivot/pivoted_rrt.sql)                                                | Renal replacement therapy.
 pivot | [pivoted_sofa](pivot/pivoted_sofa.sql)                                              | The Sequential Organ Failure Assessment (SOFA) scale.
 pivot | [pivoted_uo](pivot/pivoted_uo.sql)                                                  | Urine output.
-pivot | [pivoted_vent_setting](pivot/pivoted_vent_setting.sql)                              | Ventilator settings (tidal volume, PEEP, etc).
 pivot | [pivoted_vital](pivot/pivoted_vital.sql)                                            | Vital signs.
 **sepsis** | | Definitions of sepsis, a common cause of mortality for intensive care unit patients.
 sepsis | [angus](sepsis/angus.sql)                                                          | Sepsis defined using billing codes validated by Angus et al.

mimic-iii/concepts/diagnosis/README.md

@@ -4,7 +4,7 @@ The Clinical Classification Software (CCS) categorizes ICD-9 coded diagnoses int
 
 The `ccs_multi_dx.csv.gz` data file must be uploaded to `physionet-data.mimiciii_derived.ccs_multi_dx`.
 
-The BigQuery schema definition is available in this folder as [ccs_multi_dx.json](/ccs_multi_dx.json).
+The BigQuery schema definition is available in this folder as [ccs_multi_dx.json](ccs_multi_dx.json).
 
 ## Creation of the ccs_multi_dx.csv.gz file
 
@@ -24,7 +24,7 @@ Use Python to convert all apostrophes in `ccs_multi_dx_tool_2015.csv` into doubl
 
 ```python
 import pandas as pd
-df = pd.read_csv('ccs_multi_dx_tool_2015.csv.gz')
+df = pd.read_csv('ccs_multi_dx_tool_2015.csv')
 # remove apostrophes from header names and relabel
 df.rename(columns={"'ICD-9-CM CODE'": "icd9_code", "'CCS LVL 1'": "ccs_level1", "'CCS LVL 1 LABEL'": "ccs_group1", "'CCS LVL 2'": "ccs_level2", "'CCS LVL 2 LABEL'": "ccs_group2", "'CCS LVL 3'": "ccs_level3", "'CCS LVL 3 LABEL'": "ccs_group3", "'CCS LVL 4'": "ccs_level4", "'CCS LVL 4 LABEL'": "ccs_group4", }, inplace=True)
 

mimic-iii/concepts/diagnosis/ccs_dx.sql

@@ -1,5 +1,5 @@
 -- add in matched ID, name, and ccs_id
---  matched id (ccs_mid): the ccs ID with the hierachy, e.g. 7.1.2.1
+--  matched id (ccs_mid): the ccs ID with the hierarchy, e.g. 7.1.2.1
 --  name (ccs_name): the most granular CCS category the diagnosis is in
 --  ID (ccs_id): the CCS identifier for the ICD-9 code (integer)
 SELECT