Merge pull request #199 from populationgenomics/upstream_2024-02-06

Upstream 2024 04 22

.github/workflows/hail-search-dataset-version-release.yaml → .github/workflows/hail-search-persistent-volume-snapshot-release.yaml

@@ -1,14 +1,19 @@
-name: hail search dataset version release
+name: hail search persistent volume snapshot release
 on:
   workflow_dispatch:
     inputs:
-      reference_genome:
+      environment:
         type: choice
-        description: Reference Genome
         options: 
-        - GRCh37
-        - GRCh38
-        required: true
+        - dev
+        - prod
+      reference_genome:
+         type: choice
+         description: Reference Genome
+         options: 
+         - GRCh37
+         - GRCh38
+         required: true
       dataset_type:
         type: choice
         description: Dataset Type
@@ -20,6 +25,8 @@ on:
         required: true
       version:
         required: true
+      volume_handle:
+        required: true
 
 jobs:
   helm_update:
@@ -38,7 +45,13 @@ jobs:
         uses: mikefarah/[email protected]
         with:
           cmd: >
-            yq -i '.global.hail_search.datasetVersions.${{ inputs.reference_genome }}/${{ inputs.dataset_type }} = "${{ inputs.version }}"' charts/broad-seqr/values.yaml
+            yq -i '.global.hail_search.datasetVersions.${{ inputs.reference_genome }}/${{ inputs.dataset_type }} = "${{ inputs.version }}"' charts/broad-seqr/values-${{ inputs.environment }}.yaml
+
+      - name: update volume handle in the broad-seqr chart
+        uses: mikefarah/[email protected]
+        with:
+          cmd: >
+            yq -i '.hail-search.persistentVolume.volumeHandle = "${{ inputs.volume_handle }}"' charts/broad-seqr/values-${{ inputs.environment }}.yaml
 
       - name: Commit and Push changes
         uses: Andro999b/[email protected]
@@ -48,4 +61,4 @@ jobs:
           github_token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }}
           author_email: ${{ github.actor }}@users.noreply.github.com
           author_name: tgg-automation
-          message: "Updating ${{ inputs.reference_genome }}/${{ inputs.dataset_type }} dataset version to ${{ inputs.version }}"
+          message: "Updating ${{ inputs.environment }} ${{ inputs.reference_genome }}/${{ inputs.dataset_type }} dataset version to ${{ inputs.version }} and volume handle to ${{ inputs.volume_handle }} "

.github/workflows/hail-search-unit-tests.yaml

@@ -17,7 +17,7 @@ on:
 jobs:
   hail_search:
     runs-on: ubuntu-latest
-    container: hailgenetics/hail:0.2.126
+    container: hailgenetics/hail:0.2.128
 
     steps:
       - uses: actions/checkout@v2
@@ -29,5 +29,7 @@ jobs:
         run: |
           export DATASETS_DIR=./hail_search/fixtures
           export ONT_ENABLED=true
+          export MACHINE_MEM=24
+          export JAVA_OPTS_XSS=16M
           coverage run --source="./hail_search" --omit="./hail_search/__main__.py","./hail_search/test_utils.py" -m pytest hail_search/
           coverage report --fail-under=99

.github/workflows/prod-hail-search-release.yaml

@@ -0,0 +1,62 @@
+name: seqr hail search prod release
+on:
+  workflow_run:
+    workflows: ["Hail Search Unit Tests"]
+    types:
+      - completed
+    branches:
+      - master
+
+permissions:
+  id-token: write
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.workflow_run.head_branch }}
+
+      - name: authenticate to google cloud
+        id: "auth"
+        uses: google-github-actions/auth@v0
+        with:
+          workload_identity_provider: "${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}"
+          service_account: "${{ secrets.RUN_SA_EMAIL }}"
+
+      - name: "setup gcloud sdk"
+        uses: google-github-actions/setup-gcloud@v0
+
+      - name: Build and push images
+        run: |-
+          gcloud builds submit --quiet --substitutions="COMMIT_SHA=${{ github.event.workflow_run.head_sha }},_CUSTOM_BRANCH_TAG=gcloud-prod" --config .cloudbuild/seqr-hail-search-docker.cloudbuild.yaml --gcs-log-dir=gs://seqr-github-actions-logs/logs .
+
+  helm_update:
+    runs-on: ubuntu-latest
+    needs: docker
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: broadinstitute/seqr-helm
+          ref: main
+          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
+          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+
+      - name: Update appVersion in hail-search Chart file
+        uses: mikefarah/[email protected]
+        with:
+          cmd: >
+            yq -i '.appVersion = "${{ github.event.workflow_run.head_sha }}"' charts/hail-search/Chart.yaml
+
+      - name: Commit and Push changes
+        uses: Andro999b/[email protected]
+        with:
+          repository: broadinstitute/seqr-helm
+          branch: main
+          github_token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }}
+          author_email: ${{ github.actor }}@users.noreply.github.com
+          author_name: tgg-automation
+          message: "Update hail-search chart appVersion to ${{ github.event.workflow_run.head_sha }}"

.github/workflows/unit-tests.yml

@@ -78,7 +78,7 @@ jobs:
     - name: Run coverage tests
       run: |
         coverage run --source="./matchmaker","./seqr","./reference_data","./panelapp" --omit="*/migrations/*","*/apps.py" manage.py test -p '*_tests.py' -v 2 reference_data seqr matchmaker panelapp
-        coverage report --fail-under=90
+        coverage report --fail-under=85
 
   nodejs:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -2,6 +2,22 @@
 
 ## dev
 
+## 4/4/24
+* Add ability to import project metadata from gregor metadata
+  * Only enabled for a project if tag is first created via 
+    ```
+    ./manage.py add_project_tag --name="GREGoR Finding" --order=0.5 --color=#c25fc4 --project=<project>
+    ```
+* Support FRASER2 data (REQUIRES DB MIGRATION)
+* Add solve_status to Individual model (REQUIRES DB MIGRATION)
+* Update data deployment for hail backend to disk snapshots
+
+## 3/13/24
+* Add "Probably Solved" analysis status (REQUIRES DB MIGRATION)
+
+## 3/1/24
+* Add subscribable project notifications (REQUIRES DB MIGRATION)
+
 ## 1/8/24
 * Support OMIM entries with no associated gene and remove phenotypic_series_number (REQUIRES DB MIGRATION)
 

deploy/kubectl_helpers/logs.sh

@@ -6,4 +6,10 @@ set -x -e
 
 POD_NAME=$("${DIR}"/utils/get_pod_name.sh "$@")
 
-kubectl logs -f "${POD_NAME}" --all-containers
+COMPONENT=$2
+case ${COMPONENT} in
+  hail-search) CONTAINER="-c seqr-hail-search-pod" ;;
+  *) CONTAINER="--all-containers" ;;
+esac
+
+kubectl logs -f "${POD_NAME}" "${CONTAINER}"

deploy/kubectl_helpers/port_forward.sh

@@ -7,15 +7,9 @@ set -x -e
 COMPONENT=$2
 
 case ${COMPONENT} in
-  elasticsearch)
-    PORT=9200
-    NAME='service/elasticsearch-es-http'
-    OPEN_BROWSER=true
-    ;;
-  kibana)
-    PORT=5601
-    NAME='service/kibana-kb-http'
-    OPEN_BROWSER=true
+  hail-search)
+    PORT=5000
+    NAME='service/seqr-hail-search'
     ;;
   redis)
     PORT=6379

deploy/kubectl_helpers/set_env.sh

@@ -10,11 +10,9 @@ DEPLOYMENT_TARGET=$1
 case ${DEPLOYMENT_TARGET} in
   dev)
     CLUSTER_NAME=seqr-cluster-dev
-    NAMESPACE=gcloud-dev
     ;;
   prod)
     CLUSTER_NAME=seqr-cluster-prod
-    NAMESPACE=default
     ;;
   *)
     echo "Invalid deployment target '${DEPLOYMENT_TARGET}'"
@@ -26,4 +24,4 @@ gcloud config set core/project ${GCLOUD_PROJECT}
 gcloud config set compute/zone ${GCLOUD_ZONE}
 gcloud container clusters get-credentials --zone=${GCLOUD_ZONE} ${CLUSTER_NAME}
 
-kubectl config set-context $(kubectl config current-context) --namespace=${NAMESPACE}
+kubectl config set-context $(kubectl config current-context) --namespace=default

hail_search/constants.py

@@ -1,10 +1,14 @@
 GENOME_VERSION_GRCh38 = 'GRCh38'
+GENOME_VERSION_GRCh37 = 'GRCh37'
 
 AFFECTED = 'A'
 UNAFFECTED = 'N'
+UNKNOWN_AFFECTED = 'U'
 AFFECTED_ID = 0
 UNAFFECTED_ID = 1
+UNKNOWN_AFFECTED_ID = 2
 MALE = 'M'
+AFFECTED_ID_MAP = {AFFECTED: AFFECTED_ID, UNAFFECTED: UNAFFECTED_ID, UNKNOWN_AFFECTED: UNKNOWN_AFFECTED_ID}
 
 GROUPED_VARIANTS_FIELD = 'variants'
 GNOMAD_GENOMES_FIELD = 'gnomad_genomes'
@@ -15,6 +19,8 @@
 CLINVAR_MITO_KEY = 'clinvar_mito'
 HGMD_KEY = 'hgmd'
 STRUCTURAL_ANNOTATION_FIELD = 'structural'
+FAMILY_GUID_FIELD = 'familyGuids'
+GENOTYPES_FIELD = 'genotypes'
 
 ANNOTATION_OVERRIDE_FIELDS = [
     SCREEN_KEY, SPLICE_AI_FIELD, NEW_SV_FIELD, STRUCTURAL_ANNOTATION_FIELD,
@@ -43,20 +49,20 @@
 COMPOUND_HET = 'compound_het'
 ANY_AFFECTED = 'any_affected'
 RECESSIVE_FILTER = {
-    AFFECTED: ALT_ALT,
-    UNAFFECTED: HAS_REF,
+    AFFECTED_ID: ALT_ALT,
+    UNAFFECTED_ID: HAS_REF,
 }
 INHERITANCE_FILTERS = {
     RECESSIVE: RECESSIVE_FILTER,
     X_LINKED_RECESSIVE: RECESSIVE_FILTER,
     'homozygous_recessive': RECESSIVE_FILTER,
     COMPOUND_HET: {
-        AFFECTED: COMP_HET_ALT,
-        UNAFFECTED: HAS_REF,
+        AFFECTED_ID: COMP_HET_ALT,
+        UNAFFECTED_ID: HAS_REF,
     },
     'de_novo': {
-        AFFECTED: HAS_ALT,
-        UNAFFECTED: REF_REF,
+        AFFECTED_ID: HAS_ALT,
+        UNAFFECTED_ID: REF_REF,
     },
 }
 
@@ -68,7 +74,7 @@
 CLINVAR_PATH_RANGES = [
     (CLINVAR_PATH_FILTER, 'Pathogenic', 'Pathogenic/Likely_risk_allele'),
     (CLINVAR_LIKELY_PATH_FILTER, 'Pathogenic/Likely_pathogenic', 'Likely_risk_allele'),
-    ('vus_or_conflicting', 'Conflicting_interpretations_of_pathogenicity', 'No_pathogenic_assertion'),
+    ('vus_or_conflicting', 'Conflicting_classifications_of_pathogenicity', 'No_pathogenic_assertion'),
     ('likely_benign', 'Likely_benign', 'Benign/Likely_benign'),
     ('benign', 'Benign/Likely_benign', 'Benign'),
 ]

hail_search/deploy/Dockerfile

@@ -1,4 +1,4 @@
-FROM hailgenetics/hail:0.2.126
+FROM hailgenetics/hail:0.2.128
 
 LABEL maintainer="Broad TGG"
 

hail_search/fixtures/GRCh37/SNV_INDEL/annotations.ht/README.txt

@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2024/02/26 15:45:13

hail_search/fixtures/GRCh37/SNV_INDEL/families/F000002_2.ht/README.txt

@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+  Written with version 0.2.126-ee77707f4fab
+  Created at 2024/01/24 11:38:19

hail_search/fixtures/GRCh37/SNV_INDEL/lookup.ht/README.txt

@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2024/04/03 17:08:32

hail_search/fixtures/GRCh37/SNV_INDEL/projects/R0001_1kg.ht/README.txt

@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2024/04/03 17:41:01

hail_search/fixtures/GRCh38/MITO/annotations.ht/README.txt

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
-  Written with version 0.2.124-13536b531342
-  Created at 2023/11/22 10:50:28
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2024/02/26 14:35:40

hail_search/fixtures/GRCh38/MITO/lookup.ht/README.txt

@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2024/04/03 15:52:09

hail_search/fixtures/GRCh38/MITO/projects/R0001_1kg.ht/README.txt

@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2024/04/03 17:42:32

hail_search/fixtures/GRCh38/ONT_SNV_INDEL/annotations.ht/README.txt

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
-  Written with version 0.2.124-13536b531342
-  Created at 2023/11/21 12:19:09
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2024/03/07 14:19:33

hail_search/fixtures/GRCh38/SNV_INDEL/annotations.ht/README.txt

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
-  Written with version 0.2.115-10932c754edb
-  Created at 2023/09/11 13:37:41
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2024/02/26 15:21:48

hail_search/fixtures/GRCh38/SNV_INDEL/high_af_variants.ht/README.txt

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
-  Written with version 0.2.109-b71b065e4bb6
-  Created at 2023/08/11 14:19:35
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2024/03/04 16:14:35

hail_search/fixtures/GRCh38/SNV_INDEL/lookup.ht/README.txt

@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2024/04/03 17:00:55

hail_search/fixtures/GRCh38/SNV_INDEL/projects/R0001_1kg.ht/README.txt

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
-  Written with version 0.2.124-13536b531342
-  Created at 2023/11/27 16:03:24
+  Written with version 0.2.126-ee77707f4fab
+  Created at 2024/02/05 17:25:57

hail_search/fixtures/GRCh38/SNV_INDEL/projects/R0003_test.ht/README.txt

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
-  Written with version 0.2.124-13536b531342
-  Created at 2023/11/27 16:03:50
+  Written with version 0.2.126-ee77707f4fab
+  Created at 2024/02/05 17:29:22

hail_search/fixtures/GRCh38/SV_WES/projects/R0001_1kg.ht/README.txt

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
-  Written with version 0.2.109-b71b065e4bb6
-  Created at 2023/08/25 11:56:33
+  Written with version 0.2.126-ee77707f4fab
+  Created at 2024/02/05 17:32:17

hail_search/fixtures/GRCh38/SV_WGS/projects/R0003_test.ht/README.txt

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
-  Written with version 0.2.109-b71b065e4bb6
-  Created at 2023/08/23 14:16:26
+  Written with version 0.2.126-ee77707f4fab
+  Created at 2024/02/05 17:34:17