Merge pull request #1340 from MIT-LCP/mimic_iv_ed_v2_psql_fix

Update MIMIC-IV-ED psql build scripts to v2.0
MIT-LCP · Jul 13, 2022 · 14b9f5f · 14b9f5f
2 parents 6cf9745 + 355993c
commit 14b9f5f
Show file tree

Hide file tree

Showing 12 changed files with 449 additions and 56 deletions.
diff --git a/.github/workflows/build-db.yml b/.github/workflows/build-db.yml
@@ -53,6 +53,67 @@ jobs:
           psql -h $POSTGRES_HOST -U postgres -f mimic-iv/buildmimic/postgres/validate_demo.sql > validate_results.txt
           cat validate_results.txt
 
+        env:
+          # The hostname used to communicate with the PostgreSQL service container
+          POSTGRES_HOST: postgres
+          PGPASSWORD: postgres
+          # The default PostgreSQL port
+          POSTGRES_PORT: 5432
+          PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+
+  build-mimic-iv-ed-psql:
+    # Containers must run in Linux based operating systems
+    runs-on: ubuntu-latest
+    # Docker Hub image that `container-job` executes in
+    container: node:latest
+
+    # Service containers to run with `container-job`
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Install psql command
+        run: |
+          apt-get update
+          apt-get install --yes --no-install-recommends postgresql-client
+
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
+        with:
+            project_id: ${{ secrets.GCP_PROJECT_ID }}
+            credentials_json: ${{ secrets.GCP_SA_KEY }}
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
+
+      - name: Download demo and create tables on PostgreSQL
+        run: |
+          echo "Downloading MIMIC-IV demo from GCP."
+          gsutil -q -u $PROJECT_ID -m cp -r gs://mimic-iv-archive/v2.0/demo/ed ./
+          MIMIC_DATA_DIR=`pwd`/ed
+          echo "Building and loading data into psql."
+          cd mimic-iv-ed/buildmimic/postgres
+          psql -q -h $POSTGRES_HOST -U postgres -f create.sql
+          psql -q -h $POSTGRES_HOST -U postgres -v mimic_data_dir=$MIMIC_DATA_DIR -f load_gz.sql
+          echo "Validating build."
+          psql -h $POSTGRES_HOST -U postgres -f validate_demo.sql > validate_results.txt
+          cat validate_results.txt
+
         env:
           # The hostname used to communicate with the PostgreSQL service container
           POSTGRES_HOST: postgres

diff --git a/mimic-iv-ed/buildmimic/bigquery/schemas/ed/diagnosis.json b/mimic-iv-ed/buildmimic/bigquery/schemas/ed/diagnosis.json
@@ -1 +1,27 @@
-[{"name": "subject_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "stay_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "seq_num", "type": "INT64", "mode": "REQUIRED"}, {"name": "icd_code", "type": "STRING", "mode": "REQUIRED"}, {"name": "icd_title", "type": "STRING", "mode": "REQUIRED"}]
+[
+    {
+        "name": "subject_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "stay_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "seq_num",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "icd_code",
+        "type": "STRING",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "icd_title",
+        "type": "STRING",
+        "mode": "REQUIRED"
+    }
+]
diff --git a/mimic-iv-ed/buildmimic/bigquery/schemas/ed/edstays.json b/mimic-iv-ed/buildmimic/bigquery/schemas/ed/edstays.json
@@ -1 +1,47 @@
-[{"name": "subject_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "hadm_id", "type": "INT64", "mode": "NULLABLE"}, {"name": "stay_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "intime", "type": "DATETIME", "mode": "REQUIRED"}, {"name": "outtime", "type": "DATETIME", "mode": "REQUIRED"}]
+[
+    {
+        "name": "subject_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "hadm_id",
+        "type": "INT64",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "stay_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "intime",
+        "type": "DATETIME",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "outtime",
+        "type": "DATETIME",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "gender",
+        "type": "STRING",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "race",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "arrival_transport",
+        "type": "STRING",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "disposition",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    }
+]
diff --git a/mimic-iv-ed/buildmimic/bigquery/schemas/ed/medrecon.json b/mimic-iv-ed/buildmimic/bigquery/schemas/ed/medrecon.json
@@ -1 +1,47 @@
-[{"name": "subject_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "stay_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "charttime", "type": "DATETIME", "mode": "NULLABLE"}, {"name": "name", "type": "STRING", "mode": "NULLABLE"}, {"name": "gsn", "type": "STRING", "mode": "NULLABLE"}, {"name": "ndc", "type": "STRING", "mode": "NULLABLE"}, {"name": "etc_rn", "type": "INT64", "mode": "NULLABLE"}, {"name": "etccode", "type": "STRING", "mode": "NULLABLE"}, {"name": "etcdescription", "type": "STRING", "mode": "NULLABLE"}]
+[
+    {
+        "name": "subject_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "stay_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "charttime",
+        "type": "DATETIME",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "name",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "gsn",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "ndc",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "etc_rn",
+        "type": "INT64",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "etccode",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "etcdescription",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    }
+]
diff --git a/mimic-iv-ed/buildmimic/bigquery/schemas/ed/pyxis.json b/mimic-iv-ed/buildmimic/bigquery/schemas/ed/pyxis.json
@@ -1 +1,37 @@
-[{"name": "subject_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "stay_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "charttime", "type": "DATETIME", "mode": "NULLABLE"}, {"name": "med_rn", "type": "INT64", "mode": "REQUIRED"}, {"name": "name", "type": "STRING", "mode": "NULLABLE"}, {"name": "gsn_rn", "type": "INT64", "mode": "REQUIRED"}, {"name": "gsn", "type": "STRING", "mode": "NULLABLE"}]
+[
+    {
+        "name": "subject_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "stay_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "charttime",
+        "type": "DATETIME",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "med_rn",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "name",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "gsn_rn",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "gsn",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    }
+]
diff --git a/mimic-iv-ed/buildmimic/bigquery/schemas/ed/triage.json b/mimic-iv-ed/buildmimic/bigquery/schemas/ed/triage.json
@@ -1 +1,57 @@
-[{"name": "subject_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "stay_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "temperature", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "heartrate", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "resprate", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "o2sat", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "sbp", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "dbp", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "pain", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "acuity", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "chiefcomplaint", "type": "STRING", "mode": "NULLABLE"}]
+[
+    {
+        "name": "subject_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "stay_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "temperature",
+        "type": "NUMERIC",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "heartrate",
+        "type": "NUMERIC",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "resprate",
+        "type": "NUMERIC",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "o2sat",
+        "type": "NUMERIC",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "sbp",
+        "type": "NUMERIC",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "dbp",
+        "type": "NUMERIC",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "pain",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "acuity",
+        "type": "NUMERIC",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "chiefcomplaint",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    }
+]
diff --git a/mimic-iv-ed/buildmimic/bigquery/schemas/ed/vitalsign.json b/mimic-iv-ed/buildmimic/bigquery/schemas/ed/vitalsign.json
@@ -1 +1,57 @@
-[{"name": "subject_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "stay_id", "type": "INT64", "mode": "REQUIRED"}, {"name": "charttime", "type": "DATETIME", "mode": "NULLABLE"}, {"name": "temperature", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "heartrate", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "resprate", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "o2sat", "type": "NUMERIC", "mode": "NULLABLE"}, {"name": "sbp", "type": "INT64", "mode": "NULLABLE"}, {"name": "dbp", "type": "INT64", "mode": "NULLABLE"}, {"name": "rhythm", "type": "STRING", "mode": "NULLABLE"}, {"name": "pain", "type": "STRING", "mode": "NULLABLE"}]
+[
+    {
+        "name": "subject_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "stay_id",
+        "type": "INT64",
+        "mode": "REQUIRED"
+    },
+    {
+        "name": "charttime",
+        "type": "DATETIME",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "temperature",
+        "type": "NUMERIC",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "heartrate",
+        "type": "NUMERIC",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "resprate",
+        "type": "NUMERIC",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "o2sat",
+        "type": "NUMERIC",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "sbp",
+        "type": "INT64",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "dbp",
+        "type": "INT64",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "rhythm",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    },
+    {
+        "name": "pain",
+        "type": "STRING",
+        "mode": "NULLABLE"
+    }
+]
diff --git a/mimic-iv-ed/buildmimic/bigquery/schemas/ed/vitalsign_hl7.json b/mimic-iv-ed/buildmimic/bigquery/schemas/ed/vitalsign_hl7.json
diff --git a/mimic-iv-ed/buildmimic/postgres/README.md b/mimic-iv-ed/buildmimic/postgres/README.md
@@ -49,7 +49,7 @@ After the database exists, the schema and tables can be created under this datab
 psql -d mimic -f create.sql
 ```
 
-Finally, loading the data into this data requires specifying the database name with `-d mimicived` again:
+Finally, loading the data into this data requires specifying the database name with `-d mimic` again:
 
 ```sh
 psql -d mimic -v ON_ERROR_STOP=1 -v mimic_data_dir=<INSERT MIMIC FILE PATH HERE> -f load.sql
@@ -93,10 +93,6 @@ NOTICE:  table "XXXXXX" does not exist, skipping
 
 This is normal. By default, the script attempts to delete tables before rebuilding them. If it cannot find the table to delete, it outputs a notice letting the user know.
 
-## Older versions of PostgreSQL
-
-If you have an older version of PostgreSQL, then it is still possible to load MIMIC, but modifications to the scripts are required. In particular, the scripts use declarative partitioning for larger tables to speed up queries. To read more about [declarative partitioning, see the PostgreSQL documentation](https://www.postgresql.org/docs/10/static/ddl-partitioning.html#DDL-PARTITIONING-DECLARATIVE). You can remove declarative partitionining by modifying the create script, and removing it for each affected table. For example, chartevents in the `mimic_icu` schema uses declarative partitioning, and thus the create.sql script creates many partitions for chartevents: chartevents_01, chartevents_02, ..., etc. Replacing these with a single create statement for chartevents will make the script compatible for older versions of PostgreSQL.
-
 ### Other
 
 Please see the [issues page](https://github.com/MIT-LCP/mimic-code/issues) to discuss other issues you may be having.